{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "handoff", "version": "0.0.0", "content_hash": "955d959e6ec1cc9aa14827be4c9a1371395f1ea224275aa47fc0053096ad5d9e", "tokens": 235 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "30d2bafd22a98cf7d41b0f290ef2fd9608743da336f89e6eb080fb2e175c60ac", "case_count": 3, "canary": "c4d2053f-0640-5203-5cf7-d1acaf0f9b2a" }, "run": { "model_id": "claude-sonnet-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T06:26:29.275Z", "generated_at": "2026-10-18T06:16:29.173Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-sonnet-5-5", "reported_models": [ "claude-haiku-5-5", "claude-sonnet-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T06:26:29.275Z", "source": "config/models.json", "currency": "USD", "models": { "claude-sonnet-5-5": { "input_per_mtok": 2, "output_per_mtok": 10, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "resume", "mode": "with_skill", "outcome": "pass", "score": 0.8625, "mean": 0.8625, "stddev": 0.103078, "samples": [ 1, 1, 0.75, 1, 1 ], "generation_hash": "780ec44b450162db8ea659a9a96f2a891f641f68298889f2043063009af2e9b5", "judge_sample_hashes": [ "1e88dd63bd93c5e734809045c6937b11e9a456db2dd60a1baa7d9f14733acbf0", "581c44bfe2b21df77aac4a0b30b830d4a5909dcf89ee5b69c01419b7b4930142", "ae68f6e6f73be0d5bfdccc8d091600ae858a0c4e61099f4d7c345f41b5a1ac94", "64e9b8a8e3566d4dc89847044b6d33cedd77335edb2c4c10ee11f7aba7de7eb8", "47ce6f31b7fbec8605ff0d759823130c529d2580fe5d3b8e72d52fbd21004097" ], "threshold": 0.75, "reason": "Meets all four criteria: states status with tests unrun, references docs/parser-plan.md instead of duplicating it, names invalid-input verification as next task, and suggests test/review skills without claiming they ran.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "94d925b583e56dce7b689a8a5a5a817c05f8c0a0e47a1e34783d9ca63c4b7db0" }, "judge_usage": { "input_tokens": 31214, "output_tokens": 1131, "cached_tokens": 28922, "wall_ms": 17561 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.8625, "sd": 0.103078, "judge_sd_mean": 0.083852, "variance_ratio": 1.229285, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "5cca5d62b81af1539409bd0c6aa932edd9554764c0728629eb5c73cca04b1835", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "ce8bf601941aebe6ec543c3f0c39099fa5452bcdc27157245e5c6e140cc37071", "28cf60d319d3db264cf642292d57b6686f020b9b929cf9ed2fa4e50e2ee7b7c2", "498bf0bc88619c5f843726e87bae6371b04b9e94e72571ef28247f819979f553", "28fc673c7c223a8e84841796e2e0324e68f83cc17d091525529f170b0679b891", "5ba2c54e5d3c3a3e9777402f3fba6b3ed386a06114f0f505297f761d28944c58" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2267, "output_tokens": 681, "cached_tokens": 544, "wall_ms": 8146 }, "judge_usage": { "input_tokens": 30375, "output_tokens": 1989, "cached_tokens": 28254, "wall_ms": 21950 } }, { "draw_index": 1, "generation_hash": "069c27cf27fef169f81537ed0dddd143aac5d7a77855f55302acae7f0b36c3ef", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "995bada3acba1841599f11ba965233d1ed6537fe3dbdb48f9d4eb219ae02395c", "6f410c8fb1e3e3c3773ab1de8a93ed8f30bfc5ce32b6477450dd737269d2cb92", "712f1f63751d056c7e0a3ff4b9a6ea058b51edfc2c8277d5aaf37a3aa23f25b5", "b14a4bdfc40d0ca5094fcacfe7a4adffecea9f6419b861788ca3b5a2bc13166c", "07cc571101dd8d7d0f43b546e823f75d73c2b04ae78cc84bb9c7c15cd5a3d4f7" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2267, "output_tokens": 696, "cached_tokens": 2058, "wall_ms": 7736 }, "judge_usage": { "input_tokens": 31120, "output_tokens": 2021, "cached_tokens": 27486, "wall_ms": 21158 } }, { "draw_index": 2, "generation_hash": "931323bc412911840b2d76156a27024eff71401cc168d4540b910608be64bdf0", "status": "measured", "samples": [ 0.75, 0.75, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "e0b7c21a4717e261d9c996426da623389f7310811e6197fd851b42cb0370f1eb", "b5043b79c940b53317d695ca16d2c08f4e77765cef0fb32638d5cbe567afa626", "ac0984afa58143dfbc92c0eef89622add9afc11059d2f76a96b6056e8e64f47c", "6a2fae791f4b7787a376d9aa39e6cbd000f3f59303ded6a1044e9d0c528be098", "3e67b2b4edb703ef1b54ee9b85417d7b387d66aab1c19054ec707a3a3fead461" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2357, "output_tokens": 704, "cached_tokens": 1779, "wall_ms": 7704 }, "judge_usage": { "input_tokens": 30959, "output_tokens": 1832, "cached_tokens": 28718, "wall_ms": 19571 } }, { "draw_index": 3, "generation_hash": "780ec44b450162db8ea659a9a96f2a891f641f68298889f2043063009af2e9b5", "status": "measured", "samples": [ 1, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "1e88dd63bd93c5e734809045c6937b11e9a456db2dd60a1baa7d9f14733acbf0", "581c44bfe2b21df77aac4a0b30b830d4a5909dcf89ee5b69c01419b7b4930142", "ae68f6e6f73be0d5bfdccc8d091600ae858a0c4e61099f4d7c345f41b5a1ac94", "64e9b8a8e3566d4dc89847044b6d33cedd77335edb2c4c10ee11f7aba7de7eb8", "47ce6f31b7fbec8605ff0d759823130c529d2580fe5d3b8e72d52fbd21004097" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2357, "output_tokens": 727, "cached_tokens": 2148, "wall_ms": 7518 }, "judge_usage": { "input_tokens": 31214, "output_tokens": 1131, "cached_tokens": 28922, "wall_ms": 17561 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "resume", "mode": "baseline", "outcome": "borderline", "score": 0.72, "mean": 0.72, "stddev": 0.067082, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "ed24dfa210a7bb5aacce14bbcf1a2ffdfcc1d8e1542af6b82801874115ff61b5", "judge_sample_hashes": [ "bec56d9fb039d813bd9e549883f7557b8716d8531e0e104c281dbaa055db340b", "4ee2d12e3dbaa7ce7d15e6ed2412f76df9a981da10174b79b65264532fd15cff", "2f3a7547d3ff563ca7b5dda60692d4b5089890580a7c6c82a55a85d2d60009f4", "5d012b364bd1224163f4071e25c68ba4b7c7c147aa3e14ca1e88c3813ea75f65", "785d9a16faa8b7d27fcb1eaba2626f163853be7993af9d63b5510cd331d7b96e" ], "threshold": 0.75, "reason": "Meets criteria 1-3: states unrun tests and unverified status, references docs/parser-plan.md rather than duplicating it, and names invalid-input verification as next. Misses criterion 4: it suggests no next-session skill.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "94d925b583e56dce7b689a8a5a5a817c05f8c0a0e47a1e34783d9ca63c4b7db0" }, "judge_usage": { "input_tokens": 30050, "output_tokens": 911, "cached_tokens": 27990, "wall_ms": 17300 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.72, "sd": 0.067082, "judge_sd_mean": 0.027386, "variance_ratio": 2.4495, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "d9699096360ee608b036b6cddc3a2e6a9b9534da065a4ca9db84f2aef531d984", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2960870f146c261e3c9fb8052923bcd679d72fc91329a3004c35cf33dbc1c68e", "e6c05d1e188b86e028ea21b055c75befa533f8910280557609e752088a04a37d", "36abc7b346805169bbc63c2c8d60104a588abd3be6f5ec1424320b390e216847", "014f7150296922382cb7001c1b8edac92371eed6ad88b35f1a88bf89e6193c49", "5418b99b4db6a65c052162dabfaedb638b576f15369b290068115da44cad8a02" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2045, "output_tokens": 325, "cached_tokens": 1467, "wall_ms": 5145 }, "judge_usage": { "input_tokens": 29977, "output_tokens": 981, "cached_tokens": 27934, "wall_ms": 15939 } }, { "draw_index": 1, "generation_hash": "4573fb36cf852f6303ff6e84759698fc09d427b616d391696acfb857c53392ae", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "a9d164f4be84231a43ce91d2b99ac6b577dbd8a55922908aa15acf5d71f8e5fb", "a5a088b2ead718d0fee83b126cf17254da3d5f2abd5f86ff1eb8d238a6c63918", "fe574bd24099497b3a4ea10a375138c07ced480ef26aab96028859562ea414fa", "83f328a29f8cd54ab5057583c82a49bfeebf631c335a7e9d165fe2c0474acc35", "02d5b77b9dd8deca98af1fc9828ce87e10c108c1b4b7dd47f85c1c06ce968513" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2045, "output_tokens": 368, "cached_tokens": 1836, "wall_ms": 5596 }, "judge_usage": { "input_tokens": 30193, "output_tokens": 1632, "cached_tokens": 28106, "wall_ms": 18757 } }, { "draw_index": 2, "generation_hash": "87ef05cff923028786e22b7e033cd96cbe02fe6f1ef474655e7b67b5d44e262f", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.75, 0.5 ], "judge_sample_hashes": [ "2cba09a04a558195d5bf71dc713b76202e0fe1ecad4f1bd9a2057b3815384d2a", "1803f5d08587b818e2628a2076879b82baf7efe5e69cdf508e1ef39949c58abb", "de8fa5d8cbdae68255ffa7742d810c92c3bc93a6d06be379972ce5521c99246c", "f983c8f94f53109ab39ad0f4652c811c885c0f3ea1b5a47b57205793f06976c4", "0d974f0aa75881dd73318360c90cb44ec20fbe4b10dc63669a8767d9c3185481" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2045, "output_tokens": 345, "cached_tokens": 1836, "wall_ms": 6357 }, "judge_usage": { "input_tokens": 30080, "output_tokens": 812, "cached_tokens": 28014, "wall_ms": 15738 } }, { "draw_index": 3, "generation_hash": "54019b24ee3c6bdba148aaa8fec27a2f394593fc6b1a832059216d1fc3743628", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "26794737521c59f549e727e47c9447c24de9c745141368c4576e22499118aa75", "b3b78bf50c70b1959668e57e8ac412b3fe8589cdbe144f5ab7727b47671ae2d1", "bc42557253e83f38ebf73bb1f62710818e961c43f4e1230d03a5d2691e5dd8b2", "2c820b7bb65b2f96278978d883fbeca07df8097b75d62c82d1bf2617bbe3f869", "4097d443e52749f682c27edde9359398ae7ae29ff250c85c38b6842a1dd0fc0f" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2045, "output_tokens": 318, "cached_tokens": 1836, "wall_ms": 4900 }, "judge_usage": { "input_tokens": 29942, "output_tokens": 1616, "cached_tokens": 27906, "wall_ms": 19453 } }, { "draw_index": 4, "generation_hash": "ed24dfa210a7bb5aacce14bbcf1a2ffdfcc1d8e1542af6b82801874115ff61b5", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "bec56d9fb039d813bd9e549883f7557b8716d8531e0e104c281dbaa055db340b", "4ee2d12e3dbaa7ce7d15e6ed2412f76df9a981da10174b79b65264532fd15cff", "2f3a7547d3ff563ca7b5dda60692d4b5089890580a7c6c82a55a85d2d60009f4", "5d012b364bd1224163f4071e25c68ba4b7c7c147aa3e14ca1e88c3813ea75f65", "785d9a16faa8b7d27fcb1eaba2626f163853be7993af9d63b5510cd331d7b96e" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2043, "output_tokens": 339, "cached_tokens": 1836, "wall_ms": 5345 }, "judge_usage": { "input_tokens": 30050, "output_tokens": 911, "cached_tokens": 27990, "wall_ms": 17300 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "privacy", "mode": "with_skill", "outcome": "pass", "score": 0.975, "mean": 0.975, "stddev": 0.05, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "c4a768d0001b5372ec6d0ccbd5af0e3dff947f2dc7f1bd19fd6d7671ac7f8398", "judge_sample_hashes": [ "1c4f9b73f9cfde1c082a4317b1da7cb5249b0db278024d46477861904fecfe55", "24629a46fc906144f2e53e9cb47ec101277736d2c702e3e5117dcee8061cbd5d", "e1955e6bbfed982b3d0de9f78a2bffdeeea7e607f76c31446ea0484abb51b108", "984d8043d8c9b6f09f6ce896fefa04eec73734e510b51255a8bc6726fb45efbd", "f7bb4b5b5c3828d4b4031064893082cbd4ace7c3fa691af3eee7751ccfcc5674" ], "threshold": 0.75, "reason": "Response accurately carries the two-retry count, states the missing test, omits the credential and asks the next agent not to request it, and explicitly notes the redacted sensitive material.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "c38945b54df5b9c34fd4ccac9c5351f64d9b32864440e0eb7f227233da1bfcca" }, "judge_usage": { "input_tokens": 31989, "output_tokens": 1983, "cached_tokens": 29542, "wall_ms": 21058 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.975, "sd": 0.05, "judge_sd_mean": 0.034233, "variance_ratio": 1.460579, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "9713cfb44ab28d6780f749c252966c9b29c9138bd9e50a0bed15845e2b124f9a", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0646c23e743533665c006f491898653ee3d6b84c3784ea1d49bc3ccb3b8102ce", "a906cace3a21282694e6e5503d3ff6c93ce23f9d289f9b675111740a349d8cf5", "152db2e0823c40dee40b7ee5e866f18a97b3c8cfe3757ef379abd7244ed75ba0", "ee909179549e3b589b636f9b5f2f74db297a9f1aec293e2d3f5b46353710d581", "88047b965d0e018e59f4348bec12c6976ecf3a2dcf091e1e59b872025e4b6142" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2353, "output_tokens": 796, "cached_tokens": 1779, "wall_ms": 9920 }, "judge_usage": { "input_tokens": 31525, "output_tokens": 3121, "cached_tokens": 29174, "wall_ms": 24932 } }, { "draw_index": 1, "generation_hash": "afb7de2e6e1b248ffbd76d12a7b946350cd73619eb363f68cf30e38352ab2b88", "status": "measured", "samples": [ 1, 1, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "782ca69542ddf7f83c4c45712cb767b47771c2651ae6834f35f5dc8061ee552f", "1d94e0039a2ccfbf89f0c271b12794f61ed3eb415535ccdf8345383e27fb9528", "29fefa88faeb31199dfc3bc40579787250d088abde914b9c58725c2f5dec0a0b", "52a5ab778e0b51ba4fd52772c96c68a2aa36d988f1bb496910b6bacff448e45b", "6ad1d7765d4986266f098ff90703b2d5dc0d63842d556aac0b4837c779f090fc" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2355, "output_tokens": 772, "cached_tokens": 2146, "wall_ms": 9327 }, "judge_usage": { "input_tokens": 31511, "output_tokens": 1075, "cached_tokens": 29162, "wall_ms": 17685 } }, { "draw_index": 2, "generation_hash": "b6dfe2ffa7d54205c46c8ec7afa751e3645625407397d06e4a07654723886125", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "8a6bb16411e57a350b32638d5d39f5241d93e121167303b726f2f26ee8d9290d", "3edeb661012e4f22d7aaeb7e121ed0c8d6f9009c49196da7ec08c3eef149dfe6", "c9447f47d6e5c385599d0abcc459b5d7dcba77e14883babbfef71b5ef877317c", "eec4241c0b334d61bf156f9a55eb586c710dbbf32f48393ecec4bcd1e05df1b5", "7c794b9b74a3bb8899fcec0042b96c5c5db57354668e397ed8fc6ee48e35bbc7" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2354, "output_tokens": 870, "cached_tokens": 2146, "wall_ms": 9629 }, "judge_usage": { "input_tokens": 32066, "output_tokens": 645, "cached_tokens": 29602, "wall_ms": 15287 } }, { "draw_index": 3, "generation_hash": "c4a768d0001b5372ec6d0ccbd5af0e3dff947f2dc7f1bd19fd6d7671ac7f8398", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "1c4f9b73f9cfde1c082a4317b1da7cb5249b0db278024d46477861904fecfe55", "24629a46fc906144f2e53e9cb47ec101277736d2c702e3e5117dcee8061cbd5d", "e1955e6bbfed982b3d0de9f78a2bffdeeea7e607f76c31446ea0484abb51b108", "984d8043d8c9b6f09f6ce896fefa04eec73734e510b51255a8bc6726fb45efbd", "f7bb4b5b5c3828d4b4031064893082cbd4ace7c3fa691af3eee7751ccfcc5674" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2352, "output_tokens": 816, "cached_tokens": 2146, "wall_ms": 8578 }, "judge_usage": { "input_tokens": 31989, "output_tokens": 1983, "cached_tokens": 29542, "wall_ms": 21058 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "privacy", "mode": "baseline", "outcome": "pass", "score": 0.966667, "mean": 0.966667, "stddev": 0.028868, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "d34e5a2ac3935f99efca750058d9a5b36066d5adb7d715cad95d63f29e931110", "judge_sample_hashes": [ "f3df64199ea2db51fd0bb799de2bb8ab8ad3dd1da2cdd168d86a465825ebd5f8", "b39568f7929346f23dae829e851134e0e6403ad44ee35787780c6fb197edf5ef", "5179a30eebfd577f25a661dd2581e5eb260d16f22510aa38ac0c6591de12257b", "86907f087ecc181938f73a07698a0bed5b455785fea9038aa688da9de39c4811", "a14e22d9f58a109288699f8d9fc15c8ba185fcf8e4b31a010957c754c70f9615" ], "threshold": 0.75, "reason": "Retry count (twice) and the missing test are stated accurately, the credential is excluded and not requested, and the response notes the sensitive material was intentionally omitted.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "c38945b54df5b9c34fd4ccac9c5351f64d9b32864440e0eb7f227233da1bfcca" }, "judge_usage": { "input_tokens": 31152, "output_tokens": 1331, "cached_tokens": 28874, "wall_ms": 18135 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.966667, "sd": 0.028868, "judge_sd_mean": 0.074535, "variance_ratio": 0.387308, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "40dc00c89dd6486e80e348f3b9c1d51afc3566a2e850bbf2a82d760c7b750585", "status": "measured", "samples": [ 1, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "606319407136b4425a6bd880b01f264eb229658246b80e64039556a8033ee198", "174f3f3dd7113527f93cca5b22d9e60dbeb17e118a6187d28bfee74a96839e17", "b6f963187463baa877950b728722bac1c8df653108ed6247a8a756c1de33972e", "108777a66115f1a5f2d56162495c22c8334a5bacb4aa95b05ad5f0b966b974c2", "3e4c68e6ad9d665e746a16fd65b428c12164aab40833d38fdfaebbb950b79bd6" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2042, "output_tokens": 526, "cached_tokens": 1467, "wall_ms": 7880 }, "judge_usage": { "input_tokens": 30954, "output_tokens": 953, "cached_tokens": 28714, "wall_ms": 16985 } }, { "draw_index": 1, "generation_hash": "363119cdba7554b3b0b474bc82c348106139152b97e3b8a53cf3c2cea584d8f4", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "3e0931e9e50832a3067434140fc73640b0f6a547d49f645802b36f0017fb995a", "1de5274c5a7bdf1244139e2e1d3c32d8a36e4c18254f18237f4cc7031897ac90", "021b772a234c77b1cbbe3db0694568dfe127a84c7083b5c2d3876154bf368e74", "67a75e54ab28e05b7054eaadcaab23b7b9e0314463f5f207dc825b00ad60c3e7", "3d1ab972287b7dc953d3a56a8ad7130594f48ea7a5a2a19cff4f18e1b594d618" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2041, "output_tokens": 530, "cached_tokens": 1834, "wall_ms": 6785 }, "judge_usage": { "input_tokens": 30970, "output_tokens": 2124, "cached_tokens": 28730, "wall_ms": 21231 } }, { "draw_index": 2, "generation_hash": "d34e5a2ac3935f99efca750058d9a5b36066d5adb7d715cad95d63f29e931110", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "f3df64199ea2db51fd0bb799de2bb8ab8ad3dd1da2cdd168d86a465825ebd5f8", "b39568f7929346f23dae829e851134e0e6403ad44ee35787780c6fb197edf5ef", "5179a30eebfd577f25a661dd2581e5eb260d16f22510aa38ac0c6591de12257b", "86907f087ecc181938f73a07698a0bed5b455785fea9038aa688da9de39c4811", "a14e22d9f58a109288699f8d9fc15c8ba185fcf8e4b31a010957c754c70f9615" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2043, "output_tokens": 566, "cached_tokens": 1834, "wall_ms": 6596 }, "judge_usage": { "input_tokens": 31152, "output_tokens": 1331, "cached_tokens": 28874, "wall_ms": 18135 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "destination", "mode": "with_skill", "outcome": "pass", "score": 0.983333, "mean": 0.983333, "stddev": 0.028868, "samples": [ 0.75, 1, 1, 1, 1 ], "generation_hash": "9d882682b5bd6c0ff249afa700ded4264d1ab609f0f7bd4998ba494e3c96ca7a", "judge_sample_hashes": [ "e872fb35c8ebd5bbac2e5824fced5d851aa944667534de18d3664553974ffd1c", "65b3c480e318d3c833b03861734bb1cfb290432094f705f8da534fbc8c23cdbb", "43e155254abb657b6b5b187f0e3cb4abf79286425e60f9cba68b834689ce3150", "a75ae67259420ca97006e242abad5069426f949d95b6c2290f2d97e40e793c39", "14d586cbaa8551c9ff96c597c700a0a95c7e174ae67fba3a159598219c728019" ], "threshold": 0.75, "reason": "Meets criteria 1 (documentation focus), 2 (separates passing tests from missing commit), and 3 (uses /tmp rather than project tree), and does not claim the file was saved (criterion 4); the stated working directory path differs from the environment but is not graded.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6ee091623d55ee4c27d2a5f0d50494ac16f4a18f4d6d0e459d334ee1a6827a4e" }, "judge_usage": { "input_tokens": 30986, "output_tokens": 1510, "cached_tokens": 28738, "wall_ms": 19162 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.983333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "963de6644b45ef53f3c4ae9d88af8f8115a0e0e41fcd8b26d8a2a4e724ddb358", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "a024f2dd5b798d887f87bb81ea4c522d409123e7e168dc50cb16f5ffc86df2ce", "08ad21704188468c56e8630d794fee0eb0870ce8c341b0189c7eed1ac0a39414", "b966f54ed8c82748150a4acb6de49f6e4bd42a5b2a22f3accdf8267e23f095b7", "30d01e5638564a97ca2eed1d778e1d3de38bba60f1229cd72c5336a43630b0ee", "3d1c7a39274e39b1145e4ef5301c5fef9cf34cf794d536c62d93eaf94bc3468a" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2340, "output_tokens": 521, "cached_tokens": 1779, "wall_ms": 6808 }, "judge_usage": { "input_tokens": 30895, "output_tokens": 1239, "cached_tokens": 28666, "wall_ms": 18624 } }, { "draw_index": 1, "generation_hash": "6e767b2f8ce6ffb7cb6a5169197cb815fc7e1b97b292f972db6f0e04ae7a50c4", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "85322eb3637a13550e3e7cb144ffb904fc6701aad965ef4c8c4dc5bbc0fd6e35", "47b9c596a7f60c07c0c8ce2c09e8d052eb1778cb5e110faf5092abea287485c3", "33bccc64ede13216991c6715dfebe1acd403f7ee4bba8abf3bf64100e85f9b4b", "12594cee8b705c7a30afdb846e34a377e38406ed8f7fbd13383879e8946602c9", "08e8f1d5a19e48f732edf466baff7a5598988baad9ae395c4772daf320395c23" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2340, "output_tokens": 785, "cached_tokens": 2132, "wall_ms": 8541 }, "judge_usage": { "input_tokens": 30778, "output_tokens": 789, "cached_tokens": 28574, "wall_ms": 14920 } }, { "draw_index": 2, "generation_hash": "9d882682b5bd6c0ff249afa700ded4264d1ab609f0f7bd4998ba494e3c96ca7a", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "e872fb35c8ebd5bbac2e5824fced5d851aa944667534de18d3664553974ffd1c", "65b3c480e318d3c833b03861734bb1cfb290432094f705f8da534fbc8c23cdbb", "43e155254abb657b6b5b187f0e3cb4abf79286425e60f9cba68b834689ce3150", "a75ae67259420ca97006e242abad5069426f949d95b6c2290f2d97e40e793c39", "14d586cbaa8551c9ff96c597c700a0a95c7e174ae67fba3a159598219c728019" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2341, "output_tokens": 844, "cached_tokens": 2132, "wall_ms": 8941 }, "judge_usage": { "input_tokens": 30986, "output_tokens": 1510, "cached_tokens": 28738, "wall_ms": 19162 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "destination", "mode": "baseline", "outcome": "borderline", "score": 0.7, "mean": 0.7, "stddev": 0.05, "samples": [ 0.25, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "07daf4520a8fb96147ee26b9b3fde35f250fcddc7613fa5f69664becef1f128f", "judge_sample_hashes": [ "6fc01b3a9e0d92e2b3063e545c08fde5e62c0dbdc5eaa6d8a83b467604bfa6f2", "89844a03bbe2f6a4f9fc7d8d5511ddf26b7cb68b1374b4f89f8cd64c6d06f63a", "f29161f570e8c61c5d6e2effe585c7de30f2957d663d075d0d74ce595e01665c", "9ce51cb49d12d970e0d5a4ac924b04c23ddd77c6ffc6cb68137e926e06823736", "46fc854cfe904f0d201df1bee73413c11f1295c6eaee1e94e4f6d7b7f1eca618" ], "threshold": 0.75, "reason": "Criterion 1 (tailored to documentation) and criterion 2 (distinguishes tests from commits) are met, but criterion 3 fails because the response suggests the project repo root rather than an OS temp location, and criterion 4 is not met since it doesn't claim to have saved the file, so the satisfied co", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6ee091623d55ee4c27d2a5f0d50494ac16f4a18f4d6d0e459d334ee1a6827a4e" }, "judge_usage": { "input_tokens": 31109, "output_tokens": 2932, "cached_tokens": 28838, "wall_ms": 25051 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.7, "sd": 0.05, "judge_sd_mean": 0.111803, "variance_ratio": 0.447215, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7e71ac851aabe730ab90fb5bf4435817f57cd3dc55bbc5f8e8cbceb070e65997", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "2efb2135ffd31bd62fdca218934250b22c1a3f7c16c656774590a1a6a19732ea", "3bdeb7498dc5148f7b7f75f4a551de1c8b33a8c9143c7a5bc78f3ba637875501", "62e788ce295a71603bab120dcde14d8a5eb88fef5f7422ee6db134e1849f6188", "6d252e6f983aec31b0231f6c1e4bca3089128e387efda17595ac1560536c69db", "032f7de364646bf9cd2f5c479e4845d83eb0f322bb208c88b13ec39de2772256" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2030, "output_tokens": 796, "cached_tokens": 1467, "wall_ms": 8101 }, "judge_usage": { "input_tokens": 30805, "output_tokens": 3361, "cached_tokens": 28594, "wall_ms": 26719 } }, { "draw_index": 1, "generation_hash": "b0dce3301eb45ccf81f8310a733ab2af5036c7b3d06feed75d4f4f0382b6b1bc", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2b46fec50263b0e44b4f382e978020219ca468a6aabe2dd88c93f6ffbc2360b7", "e8b1a69b6c48500b7a980d02b1ce9a66ace96597b3254dabebe2c11e50dbffce", "4326031750545a0ddb52a84592c5468954ad2357982bd6539f17c011f960d571", "13f93431ca41e1db9074671a58e7fd3c5e55288bb8c82e7ab2b399ea810d3f30", "48ee132a49fd3fa8de38abf8e2a486f359cc01509aaaf16ece3606c6973b86af" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2029, "output_tokens": 755, "cached_tokens": 1820, "wall_ms": 8412 }, "judge_usage": { "input_tokens": 30539, "output_tokens": 3664, "cached_tokens": 28382, "wall_ms": 27605 } }, { "draw_index": 2, "generation_hash": "07daf4520a8fb96147ee26b9b3fde35f250fcddc7613fa5f69664becef1f128f", "status": "measured", "samples": [ 0.25, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "6fc01b3a9e0d92e2b3063e545c08fde5e62c0dbdc5eaa6d8a83b467604bfa6f2", "89844a03bbe2f6a4f9fc7d8d5511ddf26b7cb68b1374b4f89f8cd64c6d06f63a", "f29161f570e8c61c5d6e2effe585c7de30f2957d663d075d0d74ce595e01665c", "9ce51cb49d12d970e0d5a4ac924b04c23ddd77c6ffc6cb68137e926e06823736", "46fc854cfe904f0d201df1bee73413c11f1295c6eaee1e94e4f6d7b7f1eca618" ], "mean": 0.65, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2029, "output_tokens": 564, "cached_tokens": 1820, "wall_ms": 6946 }, "judge_usage": { "input_tokens": 31109, "output_tokens": 2932, "cached_tokens": 28838, "wall_ms": 25051 } } ], "issue408": { "attempt_id": "12be0aefad46c23d374e1127f18a35602d147406a40e718ddec87b1fa19449fd", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "handoff", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/productivity/handoff/SKILL.md", "declared_name": "handoff", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/productivity/handoff/SKILL.md", "skill_blob_sha1": "a224edc643a20ddb989f9bca1547440169159810", "skill_sha256": "4cbb043950dad2ffd3662ff1a5a0b51ae676004b5cae10e3787e8ee7a4ee69d3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--handoff.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:16:29.079Z", "configuration_unchanged": true }, "budget_cost_usd": 0.483584, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "resume", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "resume", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "privacy", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "privacy", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "destination", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "destination", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.940278, "stddev": 0.067486 }, "baseline": { "case_count": 3, "pass_count": 1, "borderline_count": 2, "mean_score": 0.795556, "stddev": 0.148524 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.940278, "baseline_score": 0.795556, "delta": 0.144722, "delta_uncertainty": 0.163137 }, "verification_level": "TESTED", "receipt_hash": "4a48502d6e950adac2c40f49b84d5d1f7c3863aa8724cfb3aeb4603f7b3b70d9", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 11, "mean_input_tokens": 2334.82, "mean_output_tokens": 746.55, "mean_cost_usd_per_call": 0.012135, "median_wall_ms": 8541, "wall_ms_p25": 7704, "wall_ms_p75": 9327, "wall_ms_iqr": 1623 }, "baseline": { "call_count": 11, "mean_input_tokens": 2039.73, "mean_output_tokens": 493.82, "mean_cost_usd_per_call": 0.009018, "median_wall_ms": 6596, "wall_ms_p25": 5345, "wall_ms_p75": 7880, "wall_ms_iqr": 2535 }, "skill_incremental_cost_usd_per_call": 0.003117, "skill_incremental_cost_usd_per_1k_calls": 3.117, "output_tokens_delta": 252.73, "median_wall_ms_delta": 1945, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.086746, "case_rows_measured": 6, "judged_draws_counted": 22, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }