{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "reddit-automation", "version": "0.0.0", "content_hash": "e159c010e92d7f1458313c6c122517c41dd853e465dcbc2ba5ece8ca47b0508f", "tokens": 1979 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "bd44fc94b915a51a8fac6d0fc84a9b2e8bcd41cf2a1c968b86e5ccce97001f88", "case_count": 3, "canary": "4d91aa17-0751-53e9-daa6-af0b86284b6c" }, "run": { "model_id": "claude-opus-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T07:47:23.663Z", "generated_at": "2026-10-18T07:34:02.482Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-opus-5-5", "reported_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T07:47:23.663Z", "source": "config/models.json", "currency": "USD", "models": { "claude-opus-5-5": { "input_per_mtok": 4, "output_per_mtok": 20, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "rank", "mode": "with_skill", "outcome": "pass", "score": 0.833333, "mean": 0.833333, "stddev": 0.028868, "samples": [ 1, 0.75, 1, 0.75, 0.75 ], "generation_hash": "084f79b26bb83751d2f4bcb242839ba86a62eb8617e2a7ee343dcd82e96566c1", "judge_sample_hashes": [ "7c70f95b34d5f7bdce8ea042f767b40920b83f8c9c021e434c70188a2ad7d8b7", "d9016a21d09b29aa9094f5902bd95f555eec0bbb1cb02361913a4fe25c5315ba", "6dc4c1228d4c70c79e2ab95d7e54c2464e147380224cd23fb5ea3149791f40ea", "0cb2e7d8427b60def33a0b01e7b6dfc1c6433b9736f0ef20ed731fd1d3214d37", "ebb8476860417e4732da62300d88f5f4059c1e5b72ca4fb9c129db22ed15dacc" ], "threshold": 0.75, "reason": "Chooses A for fresh, direct timing and product fit; excludes locked B and off-topic joke C; explains choice from the stated facts without inventing user intent.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "28bac78c134465fb5fa985bc5718ae8ce3c2e5010009abb23dfae9ec2a46d26e" }, "judge_usage": { "input_tokens": 31212, "output_tokens": 552, "cached_tokens": 28922, "wall_ms": 14947 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.833333, "sd": 0.028868, "judge_sd_mean": 0.128555, "variance_ratio": 0.224558, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7c36c576f0ba5734dcf7552931508cc395cd45ff64a972d6c34147f5d407bab4", "status": "measured", "samples": [ 0.75, 1, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "70749a42425ffea8c78ffe630aa12f5664a087026a5541f0ba4bf90b730f4d61", "e242821d6d8b36fef048976fc9365377278082b54af82878ca93442d0d8db769", "920bb4a39b441c2921504917bdf63cbf6447d5f5fd75587c7ec9e87f7e54e7a2", "43d63afa4b65acb5050740b9f479019756971b25a011542bfdb4d4158da6b0dc", "0476a18d453de44a6df1c8fc0afc7a8ca8740d28866c8560e276dc11b99c9df4" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4813, "output_tokens": 673, "cached_tokens": 544, "wall_ms": 12195 }, "judge_usage": { "input_tokens": 30995, "output_tokens": 1048, "cached_tokens": 28746, "wall_ms": 17520 } }, { "draw_index": 1, "generation_hash": "42cb62239dd59585845e47ee1970839c2d27859d7da6188b2d18e3ce303af7c0", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "887884d990d5506d566a34a9f424bf5758331998f21ba2eecbab03a2c3a9da40", "50cfd2375ea317de19936675ec600d285a6e871ad9ef6382056d8050dfb365ef", "998c16f0d48603983a759438d81d1f21238581bfe33a19dce106c70cbd99f09a", "0f4aa0cd20ebc8ac82ebf7008e96964409da2c454786de490bb144200c1454f3", "49de56fc62ace920c9cbd435df2ab1fd78eebd7d986d9a4496a105e93b8027c1" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4817, "output_tokens": 614, "cached_tokens": 4611, "wall_ms": 12605 }, "judge_usage": { "input_tokens": 31088, "output_tokens": 1677, "cached_tokens": 28822, "wall_ms": 19707 } }, { "draw_index": 2, "generation_hash": "084f79b26bb83751d2f4bcb242839ba86a62eb8617e2a7ee343dcd82e96566c1", "status": "measured", "samples": [ 1, 0.75, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "7c70f95b34d5f7bdce8ea042f767b40920b83f8c9c021e434c70188a2ad7d8b7", "d9016a21d09b29aa9094f5902bd95f555eec0bbb1cb02361913a4fe25c5315ba", "6dc4c1228d4c70c79e2ab95d7e54c2464e147380224cd23fb5ea3149791f40ea", "0cb2e7d8427b60def33a0b01e7b6dfc1c6433b9736f0ef20ed731fd1d3214d37", "ebb8476860417e4732da62300d88f5f4059c1e5b72ca4fb9c129db22ed15dacc" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4816, "output_tokens": 712, "cached_tokens": 4611, "wall_ms": 10615 }, "judge_usage": { "input_tokens": 31212, "output_tokens": 552, "cached_tokens": 28922, "wall_ms": 14947 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "rank", "mode": "baseline", "outcome": "pass", "score": 0.983333, "mean": 0.983333, "stddev": 0.028868, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "36005b89186c63c39c0341cd5f73602c852d9f3325174f6beede0b0fb2fd9865", "judge_sample_hashes": [ "96152ac757ec318f0d053cf1a53579cc354fffe22e0bc818dfada1a393aa6666", "347d9179ccbc7e345d966a2c1efe75cf4ab33696a9cfa8a0b5fa05a191e18e14", "ad4601a3a63f5bed52ae67d202a6f931832b743950b4c6897b698acab615c7f1", "de83406de13360a63bf9b565c4c88db88f77a5559ea083654b17aea89ce67987", "f3cab3786c6c8af8e560a37be9f7c19867e2553c63f3f37185f09e7cc631801a" ], "threshold": 0.75, "reason": "Response chooses A for its same-day, exact-fit offline warehouse count request (criterion 1), excludes locked B (criterion 2), excludes generic joke C (criterion 3), and explains reasoning from the supplied facts without inventing user intent (criterion 4).", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "28bac78c134465fb5fa985bc5718ae8ce3c2e5010009abb23dfae9ec2a46d26e" }, "judge_usage": { "input_tokens": 29155, "output_tokens": 802, "cached_tokens": 27274, "wall_ms": 14917 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.983333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "410f45cec5655b30a546bf4a9e518a9cc557f7fed89f7feb460519db40653531", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "4f52883a4c21ff534479ea5ef0540d79facaa9c1a505d2cc7cfa70ead63e4018", "5d3769271c998e38642743cc19f8eec54f7a802242326098c9c54e951d1683d1", "1e21440a86f8d36dfbe1c8604b03b661b269b0d332bc558e95bda020f380fd18", "14c45fa00c94a46503159e838f1249e4a5165ea4d47e8ecfb4382baea708bda9", "91e948712b69376e07251290bf698bbca03aaaa9589aadc92de310b35e83f756" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2041, "output_tokens": 257, "cached_tokens": 1467, "wall_ms": 5745 }, "judge_usage": { "input_tokens": 29618, "output_tokens": 545, "cached_tokens": 27646, "wall_ms": 15154 } }, { "draw_index": 1, "generation_hash": "19ac91393c32f5c13064e648fe500ad0777b8be12c859ef7485a36d06d3523ff", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "f58c6e7790b9436f6f6159491c01902133b6c118a580b36940c8c3de7706b571", "a47ecd7d072b3b00357ab4c1a86daf567239d5e4422fdb74e2cec89fb6c4612d", "d7e30edc385f36b1a8c7350bad8c118cb24b6a0e73bc6a074fa1a8f0de6cc8cd", "29c844c918800aa072ae569b6e2d2f9654d56ff6dce5a50af7ac4d78ea7d1a54", "d74786eb4fd5584ab7b5adf3809988ed9baa74bd79f2a05edf54cde424e47792" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2039, "output_tokens": 290, "cached_tokens": 1836, "wall_ms": 6019 }, "judge_usage": { "input_tokens": 29610, "output_tokens": 908, "cached_tokens": 27638, "wall_ms": 16426 } }, { "draw_index": 2, "generation_hash": "36005b89186c63c39c0341cd5f73602c852d9f3325174f6beede0b0fb2fd9865", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "96152ac757ec318f0d053cf1a53579cc354fffe22e0bc818dfada1a393aa6666", "347d9179ccbc7e345d966a2c1efe75cf4ab33696a9cfa8a0b5fa05a191e18e14", "ad4601a3a63f5bed52ae67d202a6f931832b743950b4c6897b698acab615c7f1", "de83406de13360a63bf9b565c4c88db88f77a5559ea083654b17aea89ce67987", "f3cab3786c6c8af8e560a37be9f7c19867e2553c63f3f37185f09e7cc631801a" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2039, "output_tokens": 164, "cached_tokens": 1836, "wall_ms": 4963 }, "judge_usage": { "input_tokens": 29155, "output_tokens": 802, "cached_tokens": 27274, "wall_ms": 14917 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "reply", "mode": "with_skill", "outcome": "borderline", "score": 0.7875, "mean": 0.7875, "stddev": 0.085391, "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "generation_hash": "9c78c91d3e7fe39af4e55afb7e919486d21b23e538a64308527209c6aaeb4147", "judge_sample_hashes": [ "b90d70d9f132b647d802f5a8fdc818cde87a24daf673ec18b32b80e6f2cd807b", "813144f3ee67c115a1d0ec518478dc7ccb968a4572394f401dba91fdaa501b0b", "4808d35ad2d10f4895d9fd6ac49d8acb2f52c7a24d9a0a7ace015aef248bb39e", "448347e1c7f20ba40b21a7d42d66aec12362253703f921051d1c1c61c3848a79", "3aa4692b3ab69a6e3c2225e0e5931e4aaba0a25670dffed2b3fbc23aeaf219e8" ], "threshold": 0.75, "reason": "Met criteria 1, 2 and 4: reacts to offline counting, discloses StockPond affiliation, and stays concise. Missed criterion 3: invents details like 'ours' and 'synced once back in range' beyond the supplied experience.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "38b200d74180a6854d1501d2c40952dae036dab5425f558b097053d37dc081d7" }, "judge_usage": { "input_tokens": 30063, "output_tokens": 1660, "cached_tokens": 28002, "wall_ms": 19992 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.7875, "sd": 0.085391, "judge_sd_mean": 0.090134, "variance_ratio": 0.947378, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "5adca90e69585700bed7cfef20fc73945c59353808a7f66059ff866a40679261", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "651d54cd8f01267ea9e3448b0dc3f3af3d52e33276ec5c5530121bbfc51e5ff1", "9c6968c990deef607911d46baf9a110ac1f21081c4ab708909e6f46325931687", "7404cd25f207e912f57d44a8caca1567e10598fd4ccd07d399fa25c2ebc96bdb", "d9470a233c93fe5ffc393d84704f1bed58875a187414f83d939ce00ba743bb2d", "d8506de12f684f17d0f03811bb4838144edd3c5f7a3cca7b50bda5bdf33d2f95" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4808, "output_tokens": 570, "cached_tokens": 4242, "wall_ms": 10359 }, "judge_usage": { "input_tokens": 29857, "output_tokens": 2564, "cached_tokens": 27838, "wall_ms": 24171 } }, { "draw_index": 1, "generation_hash": "f5e9b07beaca02366bbcd240bd99c62178f2c6ca6c484e7c058bdbbf427c3966", "status": "measured", "samples": [ 0.75, 0.75, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "76dc85de8db97bac167ffd5b9031be1bdd1150d2bba44e87a2bac106214c8131", "6e93cba97fef7bfb8fc30eefc2c04ecd7bccbfdf9568f4b8ca2b19aef1c335f0", "90df90ec2f8619fcd225122cba0c3e8a38be1fb14355e31330e1dc4dd6247816", "1504103c79abe88736586072cae663dc4e6e5249f272dbb7a3b41c9b1215d31c", "f4c30570ab4eedea438f2ca959e7f6e59b8a93ba7926fb166962186a10e2833e" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4809, "output_tokens": 389, "cached_tokens": 4604, "wall_ms": 6233 }, "judge_usage": { "input_tokens": 29704, "output_tokens": 1256, "cached_tokens": 27718, "wall_ms": 17779 } }, { "draw_index": 2, "generation_hash": "9c2429064657d931da5f694a551cbbba3a2cce74ddb0f3616d6114c537fc3f97", "status": "measured", "samples": [ 1, 0.75, 0.75, 1, 1 ], "judge_sample_hashes": [ "2ef5ea5156b01c373d850e05a07074b9d795e13f6c9200c725ceaa4c1559f008", "39316cf69b38aac1805701a578a10f621e68154a498384f89eb5223e924ca899", "9f865f5c6f7ecb0dcb0ebe5236b14cca53dcb434c860052fd3efb86e2dcf348a", "13ffe9d2ccce0c8f1db8f95253fa8f8a599ed12c417488577a3663956ac3ceea", "ab0f3024994934a3e746ef27313ca29212da153717f465455e9376c2c482de82" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4809, "output_tokens": 353, "cached_tokens": 4604, "wall_ms": 7149 }, "judge_usage": { "input_tokens": 29607, "output_tokens": 1511, "cached_tokens": 27638, "wall_ms": 18969 } }, { "draw_index": 3, "generation_hash": "9c78c91d3e7fe39af4e55afb7e919486d21b23e538a64308527209c6aaeb4147", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "b90d70d9f132b647d802f5a8fdc818cde87a24daf673ec18b32b80e6f2cd807b", "813144f3ee67c115a1d0ec518478dc7ccb968a4572394f401dba91fdaa501b0b", "4808d35ad2d10f4895d9fd6ac49d8acb2f52c7a24d9a0a7ace015aef248bb39e", "448347e1c7f20ba40b21a7d42d66aec12362253703f921051d1c1c61c3848a79", "3aa4692b3ab69a6e3c2225e0e5931e4aaba0a25670dffed2b3fbc23aeaf219e8" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4809, "output_tokens": 617, "cached_tokens": 4604, "wall_ms": 9373 }, "judge_usage": { "input_tokens": 30063, "output_tokens": 1660, "cached_tokens": 28002, "wall_ms": 19992 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "reply", "mode": "baseline", "outcome": "pass", "score": 0.87, "mean": 0.87, "stddev": 0.057009, "samples": [ 0.75, 1, 1, 0.75, 0.75 ], "generation_hash": "d83c387b94687f96be402e5392cb3eb1443e848b5ffd493b5c33748a03c9a084", "judge_sample_hashes": [ "57b3d09d4815048d66c6cb0f8efd0f9c38c522e637f2f90946f86835b09c5cfe", "67392695a2acb5a7a444a25690bb80a902e3adaf9727c8aa29da60be05fef8a6", "eb38df3721cde3f41eed4fe46b1a4207075798d4631d89f1b170163ffdd36c02", "13729edc9722733807764d83a6294f8adf96540e66bce315acd9efb3c861ddb0", "dcc943b015a3162daf0cf0a5908a70c0794e13441b6f7a9e4e8214851ed61443" ], "threshold": 0.75, "reason": "Meets criteria 1, 2, and 4: it reacts to the offline queue, discloses affiliation, and stays concise. It misses criterion 3 by adding unsupplied experience ('patchy Wi-Fi') and claiming the tool was built for 'this exact problem'.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "38b200d74180a6854d1501d2c40952dae036dab5425f558b097053d37dc081d7" }, "judge_usage": { "input_tokens": 30060, "output_tokens": 1596, "cached_tokens": 28002, "wall_ms": 19478 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.87, "sd": 0.057009, "judge_sd_mean": 0.12688, "variance_ratio": 0.449314, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "bb52461aacd752f871b0dcfb0eb1a66eba9687090f5006d984a81d954f5e9c9c", "status": "measured", "samples": [ 0.75, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "9cb6c084fd11bb4cc388bb8d25a33131588814e0e3418251c5098efe7bb77ddb", "2f977ffe7290073864af0645e1143375ba20ecb0ef4227366314fc986a34ab40", "575a387668a44e50b3ef1316339977a8693300ca04a373883862cc3922e2eefd", "c8d04639835f5d2e06cdc0b48461fb713a7cd4ccfbf63709f276cccfd7b42687", "3a00ac0f1d0a92de253bb93e3ae694e1b9dd16638e265c0aafe4aa7d28e99bed" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2032, "output_tokens": 415, "cached_tokens": 1467, "wall_ms": 10157 }, "judge_usage": { "input_tokens": 29944, "output_tokens": 954, "cached_tokens": 27906, "wall_ms": 16164 } }, { "draw_index": 1, "generation_hash": "82311767faf225ad2e61c2fb0d9016c98bf39d23aa1f51fcb4b27466de8de1d1", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "25429fe8adcfd91230e25b4997f1d233c18d270f094ec37db27da73535daf80f", "533b3ba1590575b72bfb7ff7c0b78dc56ccda77c6787130f91018489818f020e", "e71602d396f335160823bafb37ce2a99663f1ae57ba6f8c509563186150516ac", "2199db89d555c2c8eda1e09dfa54bda668c15c4097f180d97b95eab80266fa8e", "9de9928fda9b49a67fd204f06e656640ee7778ff53ca4a1fb63ceb9818ef2a21" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2034, "output_tokens": 519, "cached_tokens": 1829, "wall_ms": 8407 }, "judge_usage": { "input_tokens": 29864, "output_tokens": 1950, "cached_tokens": 27842, "wall_ms": 21562 } }, { "draw_index": 2, "generation_hash": "16db4d5c4c91324cbd2dc1b2002014c00559445f59e066cf1f7978788510e81c", "status": "measured", "samples": [ 1, 1, 1, 0.75, 1 ], "judge_sample_hashes": [ "55413e332f37171861bab89606051e66c0cdffadd2043e6e8346a679cf44dc93", "c6bde3ec67d5733e3d3e35d880b92c1c11f15b6bc206f0b4d0bc559744bfac13", "efe052541caf5a0f9b917434064452ac386814897feca1c3ab58f08a59e5fa77", "be2570c86d91fbf917c17474246b60a6424947744794217fba40de7f3eeb1274", "6a49aa8f11df3d03073b21a3d983347e41da8ab5565a72bc69b6ff7c79225ebe" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2036, "output_tokens": 490, "cached_tokens": 1829, "wall_ms": 9877 }, "judge_usage": { "input_tokens": 30097, "output_tokens": 1060, "cached_tokens": 28030, "wall_ms": 17136 } }, { "draw_index": 3, "generation_hash": "df18bed91b547cbc329dbba4b28991be443becba2f999ee055b85de4b0147ed0", "status": "measured", "samples": [ 1, 0.75, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "b6b713c2d2349596a8a072373b81e9937d47223f2d800a0856f493d790abcda3", "2e17dae573f0a8857c093ff15ad723ef368804876288f3046f9cc81f353de639", "ec333d6255ad3d5471011c603da4c410e0cd85c3432280f1745511fe87ac14ce", "bf4a3d294c7cf879edf7bb890f09d0d45f307385b63956ee7f1fd90b23844f02", "10a33ccb0a7c688089e69ba30cc7948833dbff665581763c35eeed635f1c1236" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2032, "output_tokens": 296, "cached_tokens": 1829, "wall_ms": 11141 }, "judge_usage": { "input_tokens": 29404, "output_tokens": 1843, "cached_tokens": 27478, "wall_ms": 19973 } }, { "draw_index": 4, "generation_hash": "d83c387b94687f96be402e5392cb3eb1443e848b5ffd493b5c33748a03c9a084", "status": "measured", "samples": [ 0.75, 1, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "57b3d09d4815048d66c6cb0f8efd0f9c38c522e637f2f90946f86835b09c5cfe", "67392695a2acb5a7a444a25690bb80a902e3adaf9727c8aa29da60be05fef8a6", "eb38df3721cde3f41eed4fe46b1a4207075798d4631d89f1b170163ffdd36c02", "13729edc9722733807764d83a6294f8adf96540e66bce315acd9efb3c861ddb0", "dcc943b015a3162daf0cf0a5908a70c0794e13441b6f7a9e4e8214851ed61443" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2035, "output_tokens": 475, "cached_tokens": 1829, "wall_ms": 8548 }, "judge_usage": { "input_tokens": 30060, "output_tokens": 1596, "cached_tokens": 28002, "wall_ms": 19478 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "rules", "mode": "with_skill", "outcome": "borderline", "score": 0.777778, "mean": 0.777778, "stddev": 0.130171, "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "9f721251b9a85e4dd937de47508278427313596ab4131dfaa5d38639ebdafd55", "judge_sample_hashes": [ "6ba6de52c27a60659dea43d33096aa28f5402f7c6ebf5a132b5654d0d8452377", "34c82232ea241cd8eb527e6529aec6dd0a64c1e44954ee747e2a61fcea96bd48", "1f601209aeb636ecf3ae94728e73e0b2e4dd20524df7e26c3737a2b02fb3970b", "5e874bdd664a7335672628bd3249aa409e49b763fcb5c843970113a10caa7f92", "98f80460fa27e0d6de2a5dbe47edcbbb811b2d8b59a8d6215b95eb4d9db2926e" ], "threshold": 0.75, "reason": "Response offers relevant general help (local queue sync) and does not name or link the product, meeting criteria 1 and 2. However, 'What helped in my case' in the draft implies personal experience, violating criterion 3, and it does not claim to post automatically, satisfying criterion 4 only if cou", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f8cbc02fdaa75b9309dc2e42337cadc36ec7f6a606f212d64fd7cc033ba056e3" }, "judge_usage": { "input_tokens": 30068, "output_tokens": 1499, "cached_tokens": 28002, "wall_ms": 19320 }, "generation": { "n_planned": 3, "n_drawn": 9, "n_measured": 9, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.777778, "sd": 0.130171, "judge_sd_mean": 0.080119, "variance_ratio": 1.624721, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "4d028cc84ace4618617e9e276a6859569ca8cc7cefa4432137f607980a5de66e", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "5c71b6e09a19daf659886cefd850eb1a537f975e984fe2d5bc63531de8f7e490", "059fcf17f0c176556e903a3aa578453f1a6e0106167c280b1c8be414c8864e71", "7d54ac44c73f2e03fc6e16a88cd94f3407061f645c9940369b6ea86f3724b0ba", "164a531bb395e6463c643ce33727d54476ee7756aa35cda9bce709f3d52665ec", "699b0d6093f626c25745baa24d007afc174906a0367dc5d0ba1f4b994a54c376" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4812, "output_tokens": 699, "cached_tokens": 4242, "wall_ms": 11709 }, "judge_usage": { "input_tokens": 30387, "output_tokens": 1938, "cached_tokens": 28262, "wall_ms": 24290 } }, { "draw_index": 1, "generation_hash": "af5682e5e3a9540440d537afd6d2916a2b7f0f7be84097b20f171d455698f3d2", "status": "measured", "samples": [ 1, 0.75, 1, 1, 1 ], "judge_sample_hashes": [ "be099bf0f5f46a2ebe117ce89d486349daac871ce31f0b280e3b141cdea846f4", "afbb70efa29ea8ad0965d1bdc4e1489984dbc3acdfa58f5a26f970002baa743e", "ca6be8e6dab82e60b4b9685f20a2cedcf25606b8526c600efbff9343d5a88210", "7de5570b7598eda43a2aeb4bdb3e24806afcde726b993ebbd5182faf5c547b37", "21393658cb2202f8d429a1424cefa7cd75c2367d9144083a2277e8784218fdd5" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4811, "output_tokens": 733, "cached_tokens": 4607, "wall_ms": 10864 }, "judge_usage": { "input_tokens": 30598, "output_tokens": 585, "cached_tokens": 28430, "wall_ms": 15232 } }, { "draw_index": 2, "generation_hash": "212284d21f44e53ef1296a61e0d130e1dbf470c5ca38a99c9ca6c04acf358b25", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "71dfde0217b52d1925a4cc27f56b28b397e1c5ef2755f338bf2a492275697f0d", "6faa2c45cb7061883fc554878f7c4e71f41b563b19122f3da0eb0efab70b45be", "30273792a0e4d8d6402fdefc585453c489ca63cf06197485093093ca7a988c81", "ce965dc2a22e0e78733d537ff9418084f857953911b2c96b4d8ee6bf69fc6c76", "76ea12f3383e11bc4a36c890e39f87e51b38e6447ab1f43b5cb6b78e86d70bf5" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4811, "output_tokens": 680, "cached_tokens": 4607, "wall_ms": 11166 }, "judge_usage": { "input_tokens": 30247, "output_tokens": 1469, "cached_tokens": 28150, "wall_ms": 18613 } }, { "draw_index": 3, "generation_hash": "ca38981816b076bed9b96b11ecab3f33c01146f54894480cb4456098daf6ea27", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "7cc26b11004415e3144de91c8fb5451d5a7c6ee6ad7852851b0ef68b5b2e4639", "a995fe6d68f9e188a6b3c95fb93a00895491395a0c62fb5b465a17bf5f199cec", "e136d920dfd0fa06638f686da1146e92dd4e2e7c666ec18b60b8ef6e0b22ce85", "05aa055aad4cfe866840e0edaf1b752d5bcb45a8f7627db6525b09f3ca498971", "07f405d37afefc4e44f3b0b122f747b3622bb6d3abf7aee26c4f2d431795b972" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4811, "output_tokens": 577, "cached_tokens": 4607, "wall_ms": 10328 }, "judge_usage": { "input_tokens": 30469, "output_tokens": 2468, "cached_tokens": 28326, "wall_ms": 24860 } }, { "draw_index": 4, "generation_hash": "3b9f1dac52281336e19f4d64aeefce083ae8b77f8c6bf1df483c440bb973ccb4", "status": "measured", "samples": [ 0.75, 1, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "6df0babb622ace653fc27fafff68600d25a4e89d1a72a681167ae6c1e76293e8", "a78ffc4c656fafb892a56186e2fb1c667962335fc826668bb5684daa8f9ac52a", "eca3f9a91fc1b9f2813892db93b023403dac112327063c132addefb0fcac8550", "71e53f3f150a2da8ac80eb6e7bb47ee810e374c0ee6be89382860b59a326140f", "a3f3bb01a8e2198cc9183104e16f57263b0dd939948f0ad6961707b631f41ace" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4811, "output_tokens": 762, "cached_tokens": 4607, "wall_ms": 11513 }, "judge_usage": { "input_tokens": 30122, "output_tokens": 856, "cached_tokens": 28050, "wall_ms": 16803 } }, { "draw_index": 5, "generation_hash": "8cd06a939f2e9453ded9affeb387565a7f84ad6e64242d30e95e451d9e486bef", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.5, 0.75 ], "judge_sample_hashes": [ "b5e4100547938cf056cabe22d3752eead64bab534360292bb9d82f76f19b0f73", "c8c1c7e12da0a76f888bf87f908f9c1a9bde7552b0cdd26086f126da651e1017", "ea80b3b556415ee1b4bb073afb70ec22d58a3385f4623cf623f2b424c528aa81", "5849d3dca5cf9df59ab90beb9b6289dcb565e77ccc03a671d69bed2459b18a94", "ab8b0c471777001958f780d338d9341c2dcb1319cc24cfdfcb30ff1d37bcf19b" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4812, "output_tokens": 645, "cached_tokens": 4607, "wall_ms": 9790 }, "judge_usage": { "input_tokens": 30058, "output_tokens": 1753, "cached_tokens": 27998, "wall_ms": 20189 } }, { "draw_index": 6, "generation_hash": "be4380a3a361747f678886985d7bcd625da219ed4a287c148c09679cc4a9cfcd", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "4f170acd5d1124537f121b755e3a42bfa9dcc51139eb8a3aeda7003d9f6db743", "90d04f8bc1f1b55c622766ecb93b749f62c8cc50fd7dab88fefb381f12f0a521", "deb64af3a5d33714d9ab4d6f5590e064d8eb5bf89ab4b56c372b4d43bdaf6eb4", "61ee97119c47bd79ac853ed85d1099ec37cb2e8400a6a0d535fcd1e36c1d34a1", "8e7d6b46ac8d3d2162de051b9e3f2979c61755f6ff046e083fe8afc469984356" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4812, "output_tokens": 689, "cached_tokens": 4607, "wall_ms": 9412 }, "judge_usage": { "input_tokens": 30334, "output_tokens": 1411, "cached_tokens": 28218, "wall_ms": 20254 } }, { "draw_index": 7, "generation_hash": "10dcf611b8be10ed1d94efdf665e0bb745c84ec9798c70fd5a8f0a47a2d14208", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0c4dfcea4fc226514a8f192c0646d553668cd18e9b57eb1c5123ca1af483ff22", "c2c26bc3f88a365e6eb66fbdb0c5f4761be05cfa0170647fca5c424ac4b1f497", "52b4d9af9946be0071f1567ccb61547664384c20062588a4ec56c317be701df7", "0bb895a4afca24bc19bca05a7fd2d8366ac32ba4dd1283e8d1ac7dbd6448b59b", "398da75671f44d49954e03739612baec6ee33b6130e4c1aed4fef066d11b937a" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4812, "output_tokens": 813, "cached_tokens": 4607, "wall_ms": 14210 }, "judge_usage": { "input_tokens": 30286, "output_tokens": 469, "cached_tokens": 28178, "wall_ms": 14486 } }, { "draw_index": 8, "generation_hash": "9f721251b9a85e4dd937de47508278427313596ab4131dfaa5d38639ebdafd55", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "6ba6de52c27a60659dea43d33096aa28f5402f7c6ebf5a132b5654d0d8452377", "34c82232ea241cd8eb527e6529aec6dd0a64c1e44954ee747e2a61fcea96bd48", "1f601209aeb636ecf3ae94728e73e0b2e4dd20524df7e26c3737a2b02fb3970b", "5e874bdd664a7335672628bd3249aa409e49b763fcb5c843970113a10caa7f92", "98f80460fa27e0d6de2a5dbe47edcbbb811b2d8b59a8d6215b95eb4d9db2926e" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4809, "output_tokens": 508, "cached_tokens": 4607, "wall_ms": 13442 }, "judge_usage": { "input_tokens": 30068, "output_tokens": 1499, "cached_tokens": 28002, "wall_ms": 19320 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 9 } }, { "id": "rules", "mode": "baseline", "outcome": "pass", "score": 0.8875, "mean": 0.8875, "stddev": 0.075, "samples": [ 0.75, 1, 1, 1, 1 ], "generation_hash": "cf4ece33ec56e510c776d7b329929bc8ae6566e0e06aec7c02d40b05bcee50a4", "judge_sample_hashes": [ "7ca9653f86a489118d51a1b3d142381258fba5ab43daeb5dac57d5de76f305c1", "268a2df947ff7eb11ab215de33e33e26f898d43331e74ea00ad4ffd3da7e24c5", "c11ae3aff51ed61f20bee53229454f0d7bc5cb43a2658ea930b70d1b5cfa69d3", "013acd804c49cb2e3f5e8a3e504805c0830c8974b0bb7a7c20fd33f8a8999e70", "ed7bd719b0882714b5d1c33a9e0f497db9d11c540f872ff61c1cad7dffa1cc64" ], "threshold": 0.75, "reason": "Meets criteria 1 (relevant general offline-queue help), 2 (no product named or linked), and 4 (no automatic posting claim); criterion 3 is satisfied too as no personal experience is invented. Score reflects all four criteria met, which is 1.0 under the stated 0.25-per-criterion scheme, so the score ", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f8cbc02fdaa75b9309dc2e42337cadc36ec7f6a606f212d64fd7cc033ba056e3" }, "judge_usage": { "input_tokens": 30955, "output_tokens": 490, "cached_tokens": 28714, "wall_ms": 15094 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.8875, "sd": 0.075, "judge_sd_mean": 0.118085, "variance_ratio": 0.635136, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "17ea62341f0e9662f48de2a581f28ab8e57f82a492e02243593742aaf73f0dca", "status": "measured", "samples": [ 1, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2bb71f2d981f2b9e9f54502f68eac7ababe32b37edea5accbfb480030f61cb94", "f29655e2a1cc8446d6bb7b20cc23e20881000abf1cbbe56ee05158a1ff4be8b3", "28005e132f66fffcbadd1dfab8b49507c1e5064f10b48f8d00bfe5d0700a03e7", "fb34087a447ce5a365080c24b7ad55deb3fbcdf19b40f3e9e866485e5ea58e87", "d0b9681f9054a71c78fa1a4dd4a528c5ffac43feddeaf848fe3dd3a145d930e8" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2037, "output_tokens": 636, "cached_tokens": 1467, "wall_ms": 18610 }, "judge_usage": { "input_tokens": 30948, "output_tokens": 961, "cached_tokens": 28710, "wall_ms": 17370 } }, { "draw_index": 1, "generation_hash": "1950a8310b0f50764ff9ddcfb7e3b3ba9012b6890683dc8dd54355c03e2611bf", "status": "measured", "samples": [ 1, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "3da5615a6160fd4daf90d35671dacb5bb60c1e41936896eed8a93155009d335f", "2c721d76a55ae41086a2f23cf4bcfc13b625d3e1bcb272c3717ae7bf8f3064f1", "d825f552296e86d7aafafc7ac9f9e64701f60db5e240bf519289b0543aa2cec0", "8041447bedbcf7d02998c603a03237050e02dc4a8549c9dec52123c29500ec5a", "4e5aac4fd2c059f32306a5b62235b8721b6157fd507cfd5caab84414e91a5d63" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2036, "output_tokens": 574, "cached_tokens": 1832, "wall_ms": 9366 }, "judge_usage": { "input_tokens": 30824, "output_tokens": 720, "cached_tokens": 28610, "wall_ms": 18032 } }, { "draw_index": 2, "generation_hash": "2f5493bbbf744e222596408ea8ef7a41190b1cb13becf13ee1e0b7fa748c65da", "status": "measured", "samples": [ 1, 1, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "8d4f93036b08347ad7bf99f71db47b44ab1b48690f7b239972981d4480ddcf16", "4270e03aa22b1c9f791e96975c5c6f059c680ca0350e146ba57c2f4a4ef23833", "4a4d431f7c98711578c268662d9377c5f7b7497b0a352dfe56f4ba2be17b741b", "dedb4c027ee574aeee4922c499c4c14d85b7431b399afe7f36168898f81a07a1", "3083abca1928d9e407783b0f3a6e5a66b419649d56d4029b648a5e91ec31c2bf" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2036, "output_tokens": 526, "cached_tokens": 1832, "wall_ms": 9385 }, "judge_usage": { "input_tokens": 30566, "output_tokens": 889, "cached_tokens": 28402, "wall_ms": 18153 } }, { "draw_index": 3, "generation_hash": "cf4ece33ec56e510c776d7b329929bc8ae6566e0e06aec7c02d40b05bcee50a4", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "7ca9653f86a489118d51a1b3d142381258fba5ab43daeb5dac57d5de76f305c1", "268a2df947ff7eb11ab215de33e33e26f898d43331e74ea00ad4ffd3da7e24c5", "c11ae3aff51ed61f20bee53229454f0d7bc5cb43a2658ea930b70d1b5cfa69d3", "013acd804c49cb2e3f5e8a3e504805c0830c8974b0bb7a7c20fd33f8a8999e70", "ed7bd719b0882714b5d1c33a9e0f497db9d11c540f872ff61c1cad7dffa1cc64" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2037, "output_tokens": 605, "cached_tokens": 1832, "wall_ms": 10946 }, "judge_usage": { "input_tokens": 30955, "output_tokens": 490, "cached_tokens": 28714, "wall_ms": 15094 } } ], "issue408": { "attempt_id": "3a899aa3330b1626469f4ef0106667285bda11c26c42b23f7686a7e0d7392d47", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "flowkit-labs/skills", "skill": "reddit-automation", "commit": "0c9e2b63f504a296a86e45ed3a60a426dd979ca6", "skill_path": "reddit-automation/SKILL.md", "declared_name": "reddit-automation", "skill_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/reddit-automation/SKILL.md", "skill_blob_sha1": "62ef926781dbc1503f6fe798450c8ef3c8887942", "skill_sha256": "67c1e3ce7e8209def829fa9acec3a72b7c84965c42378a4aeef32b32756ab46b", "readme_url": "https://raw.githubusercontent.com/flowkit-labs/skills/0c9e2b63f504a296a86e45ed3a60a426dd979ca6/README.md", "readme_sha256": "5c7c7f61bdd717cdb8f93e0e408ff2384d949e3c66eab379863c01c2ace2e99f", "suite": "suites/flowkit-labs--skills--reddit-automation.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:34:02.370Z", "configuration_unchanged": true }, "budget_cost_usd": 1.303894, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rank", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rank", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "reply", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "reply", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 7, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "with_skill", "draw": 8, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "with_skill", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "rules", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "rules", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 1, "borderline_count": 2, "mean_score": 0.799537, "stddev": 0.029669 }, "baseline": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.913611, "stddev": 0.061012 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.799537, "baseline_score": 0.913611, "delta": -0.114074, "delta_uncertainty": 0.067843 }, "verification_level": "TESTED", "receipt_hash": "143892731599c834f6f87b10c1dafac74e3199df65bfce3ad103313832e22bcb", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 16, "mean_input_tokens": 4811.38, "mean_output_tokens": 627.13, "mean_cost_usd_per_call": 0.031788, "median_wall_ms": 10739.5, "wall_ms_p25": 9601, "wall_ms_p75": 11952, "wall_ms_iqr": 2351 }, "baseline": { "call_count": 12, "mean_input_tokens": 2036.17, "mean_output_tokens": 437.25, "mean_cost_usd_per_call": 0.01689, "median_wall_ms": 9375.5, "wall_ms_p25": 7213, "wall_ms_p75": 10551.5, "wall_ms_iqr": 3338.5 }, "skill_incremental_cost_usd_per_call": 0.014898, "skill_incremental_cost_usd_per_1k_calls": 14.898, "output_tokens_delta": 189.88, "median_wall_ms_delta": 1364, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.102329, "case_rows_measured": 6, "judged_draws_counted": 28, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }