{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "hyperframes-animation", "version": "0.0.0", "content_hash": "8deed98cea9202e170e4f54b8d8df320af487d449f1997d4fbd0b5108c50e9d2", "tokens": 2110 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "c2b7fb8bdaedac0a7775345e0c29c8f41059f4118701acfead6e4bb4e6253546", "case_count": 3, "canary": "7fbf2336-7a9d-ba07-a002-de20d48ec6ae" }, "run": { "model_id": "claude-sonnet-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T06:47:42.524Z", "generated_at": "2026-10-18T06:36:57.275Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-sonnet-5-5", "reported_models": [ "claude-haiku-5-5", "claude-sonnet-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T06:47:42.524Z", "source": "config/models.json", "currency": "USD", "models": { "claude-sonnet-5-5": { "input_per_mtok": 2, "output_per_mtok": 10, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "arms": { "baseline": { "counts": { "generations_per_arm": 3 } } }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "timeline", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "428dec8afe5b0eb611e66bafa2278c8bf926f480f109b91a27cc92857171a6da", "judge_sample_hashes": [ "a4413b7039a2c3fc6a1767c313aa9b1b32b6a2a6af62375b547aabcc6f57503a", "916981e5178ed0db49cb13ccd6a614349e8867030e3475cc31c766e74a1c3dc8", "fda5d4f8f2bf456c97b735c4fd27dbe89887a9482323ac443fd33849235c2840", "ba9776e2b19bd7bea5f872f7da62d23c2a4e3aacb39ec2ae995ae0bfffb69d68", "694b80fb6669588b7fba73690be68fb5f7f44855213582cad8a1c6bbb3e7dd92" ], "threshold": 0.75, "reason": "Uses a single paused GSAP timeline, a small set of appropriate heading and card entrance rules, fixed fromTo values with no randomness, and no wall-clock timers or autoplay.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "91cb59393eb0abcda9171d942df77244e4fafb6ba3c7b40fa7d7425bad0cf58f" }, "judge_usage": { "input_tokens": 34282, "output_tokens": 1788, "cached_tokens": 31378, "wall_ms": 20008 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "fcf335058b6b4d53d7cabc6fb7612f8ff0d7cb5b1e9b6b40dac75b92c28057f9", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0ac514b203b4c4c10907d70d66c3d4f737b6e741b4f7365fae9435352b20be91", "a6ce7c5a9932f15efc29af9b8f4d572ae39e369a117297f4429ddb9ae1ff1123", "84a2ff987407b4bd1e2b43e8a0ecac43214e0fb764ea5af9b5d4e4b434706ba8", "3c0f267071d4ac0ddb84f812761b18e80f29a79be93fae62d6244092d7712cd0", "30d4c9c9b93fd595ce238c22300d359e434c57c9dbc1b30224448c672d877315" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4920, "output_tokens": 1438, "cached_tokens": 544, "wall_ms": 13127 }, "judge_usage": { "input_tokens": 34197, "output_tokens": 1250, "cached_tokens": 31310, "wall_ms": 17137 } }, { "draw_index": 1, "generation_hash": "dc036c094be37de821166b2f15047c88cb7bf1e95a9e050fd5e3c3727fbce2b1", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "233bb0c622e7c4419b0e424279e556e4d14d5894978eea70514215509f7420e9", "495d0465ed0694f6abf5114036dc0bb83a202d7ff76692de65741c3abed26bdc", "78597c963df73786e2e626fdaeb6049fd40af085bcdc1595271ddcb6eb682bf6", "c9efcd5cb7f462cc24e9997493a925705016d01f012bbec5cf30cf237b9e304e", "f55e2ffd7af1f20702ce59a3e7cfe1bf83669d01d01c55055650a75ec66beb73" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4921, "output_tokens": 1646, "cached_tokens": 4711, "wall_ms": 14633 }, "judge_usage": { "input_tokens": 33919, "output_tokens": 1242, "cached_tokens": 31086, "wall_ms": 16765 } }, { "draw_index": 2, "generation_hash": "428dec8afe5b0eb611e66bafa2278c8bf926f480f109b91a27cc92857171a6da", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "a4413b7039a2c3fc6a1767c313aa9b1b32b6a2a6af62375b547aabcc6f57503a", "916981e5178ed0db49cb13ccd6a614349e8867030e3475cc31c766e74a1c3dc8", "fda5d4f8f2bf456c97b735c4fd27dbe89887a9482323ac443fd33849235c2840", "ba9776e2b19bd7bea5f872f7da62d23c2a4e3aacb39ec2ae995ae0bfffb69d68", "694b80fb6669588b7fba73690be68fb5f7f44855213582cad8a1c6bbb3e7dd92" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4920, "output_tokens": 1453, "cached_tokens": 4711, "wall_ms": 12304 }, "judge_usage": { "input_tokens": 34282, "output_tokens": 1788, "cached_tokens": 31378, "wall_ms": 20008 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "timeline", "mode": "baseline", "outcome": "borderline", "score": 0.75, "mean": 0.75, "stddev": 0.05, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "11ecb32eb83647078430cdbcf2900cbce44ce16e02541b3e61834caf42456594", "judge_sample_hashes": [ "2335ac9483ce5719d5da8a7d029cc0be51d0169561fd142d05ccd533883be0c0", "dbb41208e61b30db87e4a3faffc133dd5351ee083c934d876492f4efa70db859", "21fdb2b8ed1913dbb89fadbceac4e2bdff439b5e416bdfb667fac098a75d8b76", "bd87d8cbea4b3e9e5b13570a09e5dad599585aefe52b25ea47796a15ba731982", "1461c52087a9126cd61cd52d5e03349c95b2129eb2a17677cba9096f0350b016" ], "threshold": 0.75, "reason": "Meets criteria 2 (small, fitting motion set), 3 (pure render(t) is deterministic under seeking), and 4 (bans Date.now() and separate clocks). Criterion 1 is only implied: it never says the single timeline is paused.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "91cb59393eb0abcda9171d942df77244e4fafb6ba3c7b40fa7d7425bad0cf58f" }, "judge_usage": { "input_tokens": 33473, "output_tokens": 2975, "cached_tokens": 30730, "wall_ms": 24852 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.75, "sd": 0.05, "judge_sd_mean": 0.139443, "variance_ratio": 0.358569, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "8b16c20028f6e4338770f13cf84c2be85d82163fa5c99794fd4c0d7c557eba7c", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.75, 1 ], "judge_sample_hashes": [ "b097c1c37245fbb8d88328e6a0352fc2661fd88f296ab843893f625a35c5c5f3", "9a5b9e964dba12afbfcff0cbcb0950c30e5143c0ff6bc7aa23b1b783e80ed48a", "92985b0eeacaa2486b5fdfca5241193cf56ad3837c3585849a52888f8e0b3671", "cf498fe08e6cab48a9aec8264ff452dbb92f7014453ee21b2d5e1ebae0c86856", "069d22993e137f98282c04ee508d5d266c58fbc681344c2c105358f20c0b11d0" ], "mean": 0.7, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1386, "cached_tokens": 1467, "wall_ms": 12280 }, "judge_usage": { "input_tokens": 33456, "output_tokens": 2386, "cached_tokens": 30714, "wall_ms": 23245 } }, { "draw_index": 1, "generation_hash": "22baa12fb6cb27497a7b59914c354e5a7d067b9271b712f40bf0ba46d93ae740", "status": "measured", "samples": [ 1, 0.5, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "01b64dd141bbacb974238b644a7ca7e03ed8e3654c1855dc5d71a96f88f5fafa", "356b4763f8a514727924e297e52c72d6ba3d2828b035b1bbb83c0353f3f6c03d", "89194b079edb244b250252e9ecdb209338c556e17b3f879a8f572d232371711f", "8c4b760ee640fac5180a7fc475afc4c7037664b89e3390ac95fff8a7b94dc07d", "e776321492cbe916009b31937ae13ad1a9107aade23eb85210f61260589ae835" ], "mean": 0.8, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1300, "cached_tokens": 1811, "wall_ms": 12072 }, "judge_usage": { "input_tokens": 33165, "output_tokens": 2709, "cached_tokens": 30486, "wall_ms": 23745 } }, { "draw_index": 2, "generation_hash": "11ecb32eb83647078430cdbcf2900cbce44ce16e02541b3e61834caf42456594", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2335ac9483ce5719d5da8a7d029cc0be51d0169561fd142d05ccd533883be0c0", "dbb41208e61b30db87e4a3faffc133dd5351ee083c934d876492f4efa70db859", "21fdb2b8ed1913dbb89fadbceac4e2bdff439b5e416bdfb667fac098a75d8b76", "bd87d8cbea4b3e9e5b13570a09e5dad599585aefe52b25ea47796a15ba731982", "1461c52087a9126cd61cd52d5e03349c95b2129eb2a17677cba9096f0350b016" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1320, "cached_tokens": 1811, "wall_ms": 12310 }, "judge_usage": { "input_tokens": 33473, "output_tokens": 2975, "cached_tokens": 30730, "wall_ms": 24852 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } }, { "id": "runtime", "mode": "with_skill", "outcome": "fail", "score": 0.64, "mean": 0.64, "stddev": 0.065192, "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "generation_hash": "04edecfcee70f2fe14c6c5942e62b3b1ef423ba4e2266cf2e7771a5630052c60", "judge_sample_hashes": [ "3fb8d13283779e3287b4088be611f871ba1107d2d82d6004ff771957dab30d09", "097cbfd3d5e5fd7170711cf327254a8a9f498c319f74fffa42810acf4251a922", "486e7dba4b304c85954d5bb957088887136b77175a808d7bd2a7641abac4e42a", "844669b325b3bf46f521855307222af3bc8828ccf42c383f687b26520d75e2c1", "d432424735b569f8f861569271404948278eceda271f76ae94ba8bbdb25be9d4" ], "threshold": 0.75, "reason": "Routes to adapters/lottie.md and ties playback to the seek-driven timeline via window.__hfLottie, but never explicitly checks repeatable seeking and never states not to replace the renderer.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "fcd043ddff30113d193484fd1290ad1a65ae40a9e270a610d08306ec4e9bdfc9" }, "judge_usage": { "input_tokens": 29686, "output_tokens": 3838, "cached_tokens": 27702, "wall_ms": 29719 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.64, "sd": 0.065192, "judge_sd_mean": 0.121854, "variance_ratio": 0.535001, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "e0e750cba7f8dbe35b8c474cafb60e0f52024c055eebd723d8ecb040030eadc4", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "c4ca99d61b88e1f386d75bbb2aa0dfc49a96dc44e2bff46a712fcc1f85832c57", "146391e2fecd28a1f08a72f3f039c08e1cdb3c86f3fcbb297ae350e815eae4bf", "f63fefdfc03166910c828df567e6725ca5011d948fcc87414541d0d31257ba0a", "8512a1ad359eb50d28fc4be7959163cb76dfb428007f1b4eb2d43a3dd5fbd660", "4aac9e78df84ff5d1fe0af94fe757491c3389810bfcaa9d2eb69eff24e4ac6c1" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4922, "output_tokens": 392, "cached_tokens": 4367, "wall_ms": 5721 }, "judge_usage": { "input_tokens": 29519, "output_tokens": 3496, "cached_tokens": 27566, "wall_ms": 27089 } }, { "draw_index": 1, "generation_hash": "a40b5dc233cdaaad1965d8c1beb239d0194b191e578f2f464049cf6579af3b7f", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "8c1d7381d8d64f07511be5a889534a378e594d757fe7265c205bfa11bd54ffb5", "15806593033d9047988b021b2b7a83dfbf95302a333669eb2ae28da9dec4aa55", "b0fa34c0bcb58efb160fc893e8f53d83df71bb0c3eff374400561ef6531451b7", "273b089efaa2aab7dbfd4a1673a495e0af57d86b460fa733c36c0c89366ed92f", "6176300eec68be54e6de4c95ded75b12268fbea7eecf589f170b4fc712376313" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4923, "output_tokens": 384, "cached_tokens": 4714, "wall_ms": 5063 }, "judge_usage": { "input_tokens": 29616, "output_tokens": 3803, "cached_tokens": 27642, "wall_ms": 28010 } }, { "draw_index": 2, "generation_hash": "b9c3f1fbe4d0df9fcb110b5bfd9e5ac19e0394ebc116e0f3e64f20c8743e89cb", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "45e16e79c513ca4c900cee1995d0d7e4d9a2658d6002b00651e40bf046e18d39", "f690e3f2f5cb06ee4361122bfadfe49109fde4a85145a4af65712b24c7223882", "d9aa63805e9ae3411843230b28ffe2edcc01077799a8c50649a923cf5493fe09", "ddad739a1080bb56f9d2ff31e0739ed4cb17d8554a2f0fc4264c9c024219b3b4", "9a932d62b61d954df7b4a786c90e8a1bb2548d9698f284dc64532f0495568351" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4921, "output_tokens": 500, "cached_tokens": 4714, "wall_ms": 6112 }, "judge_usage": { "input_tokens": 29544, "output_tokens": 4340, "cached_tokens": 27586, "wall_ms": 29540 } }, { "draw_index": 3, "generation_hash": "8ebfcbfbd02b24343e7fb681d62262dd6699f1f0a2283a16de67437444bded51", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.5, 0.75 ], "judge_sample_hashes": [ "b32adebaa17e01ab126f52c866e6bb6165ae00049c1f6b4b23652e7804126b2e", "07283d5d92b7d5a83e595b12438554a1e389fb35b8206a25d04a1e3e91a7396f", "f141583d0348d004f726955db04928e6d333667a8fe51a599c4d112dab300434", "ded23a95cb0f693c8c6081b99adf830f9b8b1146b6c21fa15cca53d49cc860bb", "4eda25253efc51e8595e2b9ce66d5a931f8824f17e283e642c4c565a4298ffe5" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4923, "output_tokens": 425, "cached_tokens": 4714, "wall_ms": 6078 }, "judge_usage": { "input_tokens": 29532, "output_tokens": 3152, "cached_tokens": 27578, "wall_ms": 26198 } }, { "draw_index": 4, "generation_hash": "04edecfcee70f2fe14c6c5942e62b3b1ef423ba4e2266cf2e7771a5630052c60", "status": "measured", "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "3fb8d13283779e3287b4088be611f871ba1107d2d82d6004ff771957dab30d09", "097cbfd3d5e5fd7170711cf327254a8a9f498c319f74fffa42810acf4251a922", "486e7dba4b304c85954d5bb957088887136b77175a808d7bd2a7641abac4e42a", "844669b325b3bf46f521855307222af3bc8828ccf42c383f687b26520d75e2c1", "d432424735b569f8f861569271404948278eceda271f76ae94ba8bbdb25be9d4" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4922, "output_tokens": 440, "cached_tokens": 4714, "wall_ms": 5821 }, "judge_usage": { "input_tokens": 29686, "output_tokens": 3838, "cached_tokens": 27702, "wall_ms": 29719 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "runtime", "mode": "baseline", "outcome": "fail", "score": 0.2, "mean": 0.2, "stddev": 0.05, "samples": [ 0.25, 0.25, 0.5, 0.25, 0 ], "generation_hash": "eaa4792ee381c906212a21012acdd35b94699818a1b14c455691f2273645a734", "judge_sample_hashes": [ "91fbf2422f6f777bd2ccb5921874a0c79d05bf45810c98846a82e11252a8b6fc", "b51bdfaa81a3bec0b8722edd85b91c973d3ce36b1daeeb946eca15eab7f4dc3a", "a1fbd7f13948dfbea8a52fb161507301a1f0bf0409684ba1e75504b036c8f458", "bd543e9fb0969aa083d053fe5b065ebe8ad1ec38c5b3172bd4de699641d4ab37", "c0eb0a748cd4687cc207e624131e8b66776edf470faf5c3334f9114cbde30970" ], "threshold": 0.75, "reason": "Meets only point 4 by not replacing the renderer. Misses routing to the Lottie adapter, connecting playback to the composition timeline, and checking repeatable seeking; it cites generic lottie-web docs instead.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "fcd043ddff30113d193484fd1290ad1a65ae40a9e270a610d08306ec4e9bdfc9" }, "judge_usage": { "input_tokens": 30022, "output_tokens": 2742, "cached_tokens": 27970, "wall_ms": 24798 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.2, "sd": 0.05, "judge_sd_mean": 0.141837, "variance_ratio": 0.352517, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "453e5644ac57cf44c3bb41bec49ea7d3ddbf475571b5f7ee8f347be4f1de1c70", "status": "measured", "samples": [ 0.25, 0.25, 0.25, 0, 0 ], "judge_sample_hashes": [ "2b96d5cde33e5366f1f4e5011ca7f023db31ff9403ea14d7ec3482d2c3f60ec0", "a274053c271af75f7987dfd26b2d2edc30a7549e9bcdea9c39c7b1487d5623c7", "2e3e1920bfe23e4e244fad0f347c85507ec4dd448eece6efc718d6f107087463", "3b685a580c9d9433916f0c0fcdea649c52b88e33bb3f543b3c9d336a2e81367a", "965b5424003d4767fb1fdc469d048e77bc0dd4441361f0600099276e275d484a" ], "mean": 0.15, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2023, "output_tokens": 650, "cached_tokens": 1467, "wall_ms": 7358 }, "judge_usage": { "input_tokens": 30285, "output_tokens": 2615, "cached_tokens": 28182, "wall_ms": 23850 } }, { "draw_index": 1, "generation_hash": "6ad098e458d02fdd08c8de769b5779cb3bb1b357285c4c89f0f2f64f251bb9e9", "status": "measured", "samples": [ 0.25, 0, 0.25, 0.25, 0.25 ], "judge_sample_hashes": [ "0a55de5a159114da48cbae8f4b4f622a55d7fd2b2f09a1466c6faf6db01dcb1a", "70c99b7b26c65bae9a3bbb238ecc8739d5961cc038dd63360425e604ecb1c9c8", "a80a2403591e48ee44e51a91a61631c879c6498f4ff8688dbdd1164c5933a2f7", "6650e767d25c85f47ee1830978c8bca3d2b1dde2d0bfb4b46b079ef91a5f87e6", "9f8c92b58fda94b2c3daf72fc42b2bf3188fabc5fc125a65024d76aebedccc72" ], "mean": 0.2, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2021, "output_tokens": 724, "cached_tokens": 1814, "wall_ms": 8503 }, "judge_usage": { "input_tokens": 30244, "output_tokens": 3275, "cached_tokens": 28146, "wall_ms": 26760 } }, { "draw_index": 2, "generation_hash": "eaa4792ee381c906212a21012acdd35b94699818a1b14c455691f2273645a734", "status": "measured", "samples": [ 0.25, 0.25, 0.5, 0.25, 0 ], "judge_sample_hashes": [ "91fbf2422f6f777bd2ccb5921874a0c79d05bf45810c98846a82e11252a8b6fc", "b51bdfaa81a3bec0b8722edd85b91c973d3ce36b1daeeb946eca15eab7f4dc3a", "a1fbd7f13948dfbea8a52fb161507301a1f0bf0409684ba1e75504b036c8f458", "bd543e9fb0969aa083d053fe5b065ebe8ad1ec38c5b3172bd4de699641d4ab37", "c0eb0a748cd4687cc207e624131e8b66776edf470faf5c3334f9114cbde30970" ], "mean": 0.25, "stddev": 0.176777, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2022, "output_tokens": 703, "cached_tokens": 1814, "wall_ms": 7974 }, "judge_usage": { "input_tokens": 30022, "output_tokens": 2742, "cached_tokens": 27970, "wall_ms": 24798 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } }, { "id": "template", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "bd54a70298283e583d10ef0132ed9c7f39316c2385a7f921148a3976340e74e8", "judge_sample_hashes": [ "5f23477dbe8bbb4a73880292d27f524d82b6e1538205151d7118cb50e7560de5", "b46cd1fc110ecd67c79498f77bb3d80e755a127b0ea10e8372663f3a2c39de73", "bd057db706d1ffba3ef94f062e1dac0ad9a8c18e72cf66d2a27380bdf8130ffb", "f176e1a9674c0794975e386018db68faa83d79fe5d8f42d2d4ea0bc06b94ae16", "c0c0c9f1fb36d8e030426522cecebafc36922711f9672863c9f9863f03a9fa7b" ], "threshold": 0.75, "reason": "Recommends the matching brand-reveal blueprint for multi-phase choreography, describes atomic rules as building blocks, keeps the single paused seek-safe timeline constraints, and says not to read unrelated blueprint files speculatively.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "d7c2df4e4aaebffd2ab3e7fbd76d39170e966a2643313aa131b8561c508a4bc5" }, "judge_usage": { "input_tokens": 31749, "output_tokens": 1985, "cached_tokens": 29350, "wall_ms": 19602 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "85f36702287bee59ce71cc1d3a62a59f051779c044f4acddd48381fc94ff0251", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "cb55368982aa2eb92223e25b7f36269375d1edc947301cf3bed9348eeb99d9c3", "86ae699dd397dec07d39dc58ec99bbbeaebf9d4172aeb86308e0193409b1c2c3", "9dc63d610bc5b47ac370c535dce57f28e6e69e7c3c77ac9159c14260bee59b91", "0a7e75f0ad5eb053ac0ff94939268bde840b12c2e1eacee1e7680509f278cc97", "45b5b5cf7167ff879c91b4c266ed73c52ee241fb4d1aacf66221807e35719d93" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4924, "output_tokens": 699, "cached_tokens": 4367, "wall_ms": 7163 }, "judge_usage": { "input_tokens": 31869, "output_tokens": 1945, "cached_tokens": 29446, "wall_ms": 20277 } }, { "draw_index": 1, "generation_hash": "325e3b14cfb2e6912828a9fe6e1b470b0632bb7aafacda0196cf02d52560b542", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "2c508061df02623a72b5cc88a9bb2272438c7d6539fb1fbb4b9e6bbc1bd359ce", "2aa48dd39bbc1ca538475b0320ccd15edf45d041f5ccef8bc6cefade0830797e", "8c6a248a3c37e851743792a609fbfbb5e8c5fb81fabdc9b643ef2473e348fafd", "f9eb8d72cc0a4735aa411e6396d6eb2321c5e80811094e61580588f21a311eb3", "705e1e1274f5cb0510f02584977f0ec9951a14d4178f26bc04442fbbd6ec6ffc" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4924, "output_tokens": 707, "cached_tokens": 4715, "wall_ms": 6818 }, "judge_usage": { "input_tokens": 31906, "output_tokens": 1986, "cached_tokens": 29478, "wall_ms": 20361 } }, { "draw_index": 2, "generation_hash": "bd54a70298283e583d10ef0132ed9c7f39316c2385a7f921148a3976340e74e8", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "5f23477dbe8bbb4a73880292d27f524d82b6e1538205151d7118cb50e7560de5", "b46cd1fc110ecd67c79498f77bb3d80e755a127b0ea10e8372663f3a2c39de73", "bd057db706d1ffba3ef94f062e1dac0ad9a8c18e72cf66d2a27380bdf8130ffb", "f176e1a9674c0794975e386018db68faa83d79fe5d8f42d2d4ea0bc06b94ae16", "c0c0c9f1fb36d8e030426522cecebafc36922711f9672863c9f9863f03a9fa7b" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4924, "output_tokens": 675, "cached_tokens": 4715, "wall_ms": 6701 }, "judge_usage": { "input_tokens": 31749, "output_tokens": 1985, "cached_tokens": 29350, "wall_ms": 19602 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "template", "mode": "baseline", "outcome": "fail", "score": 0.566667, "mean": 0.566667, "stddev": 0.028868, "samples": [ 0.75, 0.5, 0.5, 0.5, 0.5 ], "generation_hash": "337551288f6094202ca321a54fa437774266261777b842a27753185b2694f11e", "judge_sample_hashes": [ "592397490eea6fe4401b0e2d6046eda389ca67a0e690ba22e62a5c847f327987", "00eb92dcc942180592ffdc9ce67724ab1dd22d40a23a3458b4bff999124aa86d", "b4556f94ce356c335d900c39c9f9ead299752264505d2ba9f9c9848cfa626b50", "03ba7f764fc0531fb165292e2771c397ce10422c0d0b11d5abce36293ffb8b69", "fce17c45d845b1d9be616f14acc863736f0623fcbb0ac4e19e1c6314fe286b51" ], "threshold": 0.75, "reason": "Meets blueprint-for-multi-phase-choreography and atomic-rules-as-smaller-blocks, and implicitly avoids unneeded templates. Misses the seek-safe timeline contract entirely.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "d7c2df4e4aaebffd2ab3e7fbd76d39170e966a2643313aa131b8561c508a4bc5" }, "judge_usage": { "input_tokens": 32009, "output_tokens": 2483, "cached_tokens": 29558, "wall_ms": 22274 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.566667, "sd": 0.028868, "judge_sd_mean": 0.120179, "variance_ratio": 0.240208, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7d44f86047cd62ad0dc2edd4b05e72885008e04c6eca53a0f7fe9d8d6c402ffa", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.75, 0.5 ], "judge_sample_hashes": [ "e503bcf4c7496588ad0d88c13facfbaea4820ec788d034e6d0ff99eb6cd9b3c1", "6e698c6eb981691896eeee8cb3bf441ac07132f03839eb66b5c75a09bc918fc6", "2870c1bb21cbf776fa307cb3c6f976cfaf8455399df82ec34dddb4020ac4dbe2", "76e18d968ed4191648a835f456e47046560c99ff686e65b3c622fc9e0b59373b", "56e5eedc916d73dff65f9416b0186de65d5048a6b3bcd654f57bfbba957d8960" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2024, "output_tokens": 900, "cached_tokens": 1467, "wall_ms": 10131 }, "judge_usage": { "input_tokens": 32305, "output_tokens": 3060, "cached_tokens": 29794, "wall_ms": 23768 } }, { "draw_index": 1, "generation_hash": "99aee16c8385f11ad2b0fbfe540e4b62a4f95fc54cdcc360edb521443c1f71d2", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "c7fd142a58f3b54039fd4594cbdb58ae0f4c28fb0d220482d15fdb3d139b3e74", "8aafcea6d891477324948d95980d706b2ec9f9c81c04eb1c8fc79c1cef4d61bb", "c8c805ddddef6291e1bd12dd56f3f9260f5aaabf7e3eba587c8bc8ed7cd10608", "999027c44594fcd049d13e365e2f26c38b69ff1ce65fc5a9d2b083c9a3ac6048", "e1793e87a288f04e75510ac1b7a075240d392da5898987813fb5b1a6fc0588f7" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2023, "output_tokens": 765, "cached_tokens": 1815, "wall_ms": 8528 }, "judge_usage": { "input_tokens": 31615, "output_tokens": 1655, "cached_tokens": 29246, "wall_ms": 18888 } }, { "draw_index": 2, "generation_hash": "337551288f6094202ca321a54fa437774266261777b842a27753185b2694f11e", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "592397490eea6fe4401b0e2d6046eda389ca67a0e690ba22e62a5c847f327987", "00eb92dcc942180592ffdc9ce67724ab1dd22d40a23a3458b4bff999124aa86d", "b4556f94ce356c335d900c39c9f9ead299752264505d2ba9f9c9848cfa626b50", "03ba7f764fc0531fb165292e2771c397ce10422c0d0b11d5abce36293ffb8b69", "fce17c45d845b1d9be616f14acc863736f0623fcbb0ac4e19e1c6314fe286b51" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2024, "output_tokens": 847, "cached_tokens": 1815, "wall_ms": 9321 }, "judge_usage": { "input_tokens": 32009, "output_tokens": 2483, "cached_tokens": 29558, "wall_ms": 22274 } } ], "issue408": { "attempt_id": "d3d22183f51e5d3463cf278678a3579ca8fe8e0573484c03c103289361eaac79", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-animation", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-animation/SKILL.md", "declared_name": "hyperframes-animation", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-animation/SKILL.md", "skill_blob_sha1": "e5299de70e3c7339fcc4b8870f195b6f2e0069af", "skill_sha256": "b35f1f8eff68e9ff480a5d5d343a1bf5a40ebb7d0d9be20b0b980cfdb36047f3", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-animation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:36:57.180Z", "configuration_unchanged": true }, "budget_cost_usd": 0.615746, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "runtime", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "runtime", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "template", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "template", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 2, "borderline_count": 0, "mean_score": 0.88, "stddev": 0.207846 }, "baseline": { "case_count": 3, "pass_count": 0, "borderline_count": 1, "mean_score": 0.505556, "stddev": 0.280046 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.88, "baseline_score": 0.505556, "delta": 0.374444, "delta_uncertainty": 0.348749 }, "verification_level": "TESTED", "receipt_hash": "b9f499c15c4f83c9a4e1171e88681271c6d6d2cc8637bac97659c5c05a6ab0ce", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 11, "mean_input_tokens": 4922.18, "mean_output_tokens": 796.27, "mean_cost_usd_per_call": 0.017807, "median_wall_ms": 6701, "wall_ms_p25": 5821, "wall_ms_p75": 12304, "wall_ms_iqr": 6483 }, "baseline": { "call_count": 9, "mean_input_tokens": 2021.89, "mean_output_tokens": 955, "mean_cost_usd_per_call": 0.013594, "median_wall_ms": 9321, "wall_ms_p25": 8238.5, "wall_ms_p75": 12176, "wall_ms_iqr": 3937.5 }, "skill_incremental_cost_usd_per_call": 0.004213, "skill_incremental_cost_usd_per_1k_calls": 4.213, "output_tokens_delta": -158.73, "median_wall_ms_delta": -2620, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.089602, "case_rows_measured": 6, "judged_draws_counted": 20, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }