{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "agent-browser", "version": "0.0.0", "content_hash": "3263f7968d97e5676185ca3135b19feda782671f6a0b81f87cff9df67dbe536e", "tokens": 864 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "f4c50a9752688546d86043fd27f2c467e2c18aad5b81b0b697dea3aba506c662", "case_count": 3, "canary": "2f6e31e4-462a-9b0d-fc26-d8a24ce627dd" }, "run": { "model_id": "claude-sonnet-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T06:16:28.862Z", "generated_at": "2026-10-18T06:03:36.570Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-sonnet-5-5", "reported_models": [ "claude-haiku-5-5", "claude-sonnet-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T06:16:28.862Z", "source": "config/models.json", "currency": "USD", "models": { "claude-sonnet-5-5": { "input_per_mtok": 2, "output_per_mtok": 10, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "web", "mode": "with_skill", "outcome": "pass", "score": 0.983333, "mean": 0.983333, "stddev": 0.028868, "samples": [ 1, 1, 1, 1, 0.75 ], "generation_hash": "244e7a88f0b6eea6f35a2d11579fe5e6782821a0606e9ddf28242610e3ef58f0", "judge_sample_hashes": [ "2a97a6b64a771a099b7fbae1a334f9caf153101e805768c872e928d06942c277", "6dfba43c43802167fbe6bddb491878998230324a1b166809a461cea98fe63c8f", "966f3878a1a62081f8779960742752689d4d4b928f81f03a546be4d732113c14", "95bdb7fd74d71ef1b30045f5d3f2c06facc556e65fae9de558eef9e13914ef2c", "bd11e3d2ad894f6b4e7fd9057553e3ba4f5bed117d97dc88eac2b4d84daf71e6" ], "threshold": 0.75, "reason": "Criterion 1 met: it loads the core guide first via 'agent-browser skills get core'. Criterion 2 met: it says the snapshot is needed before choosing refs. Criterion 3 met: it explicitly refuses to assume ref numbers. Criterion 4 met: it states nothing was run and no click or submission occurred.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "ac340f784f1bb360d7df89c85147cdde0601842faff9ac8daf5a59047b66076b" }, "judge_usage": { "input_tokens": 31259, "output_tokens": 1372, "cached_tokens": 28958, "wall_ms": 19024 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.983333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7990e89f583081504e64931f959996517fcfc2ab3b4204019a0d67cbc938cd15", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "14c4a2a5fb5f2509855704179c14961abba8f77455807ae44392b8c0badbcfd5", "9e183c3c45e120bdb46b76a51eb8443bfff4d354a4d859350131adec64d0f5e2", "5a0955dedb1b5a906c7b19f972f3f97da0f9adf58382f4b423e65ae7c6c6866c", "1802592ca4ae07015f90f87e2e0b94d948e948d04f7e30d18ea41e309eb7bf24", "b65de2c84c04022d784687a577adf4da45a58df89e2d9f501829167121d428a3" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3173, "output_tokens": 758, "cached_tokens": 2964, "wall_ms": 8653 }, "judge_usage": { "input_tokens": 31001, "output_tokens": 888, "cached_tokens": 28750, "wall_ms": 16770 } }, { "draw_index": 1, "generation_hash": "d00e85209756fc7588e0e1eec2107960b8c5b8ce668c8c772b23a182888fb69b", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "bd863510cdc8662e833ab08bc8be3ad99f40a220e9666d4d005e27613d553f2f", "4db987c324d727fc6c540ed207de86fd251ec08dc04dde0e38668ea222eb297d", "b2148309361470dd294cb714f7432b29f7549e09066f93287fdd17e40a632b1f", "0902004984bbe549b524aec3ac46a97cc0d4e4a4cac65a8b1e3f21d0c5081d80", "de30ce6dbf94765cc34bda6b5207b9cfb1243be5f874778c6426279114ce9651" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3174, "output_tokens": 794, "cached_tokens": 2964, "wall_ms": 9034 }, "judge_usage": { "input_tokens": 31171, "output_tokens": 1543, "cached_tokens": 28890, "wall_ms": 18489 } }, { "draw_index": 2, "generation_hash": "244e7a88f0b6eea6f35a2d11579fe5e6782821a0606e9ddf28242610e3ef58f0", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "2a97a6b64a771a099b7fbae1a334f9caf153101e805768c872e928d06942c277", "6dfba43c43802167fbe6bddb491878998230324a1b166809a461cea98fe63c8f", "966f3878a1a62081f8779960742752689d4d4b928f81f03a546be4d732113c14", "95bdb7fd74d71ef1b30045f5d3f2c06facc556e65fae9de558eef9e13914ef2c", "bd11e3d2ad894f6b4e7fd9057553e3ba4f5bed117d97dc88eac2b4d84daf71e6" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3173, "output_tokens": 895, "cached_tokens": 2964, "wall_ms": 9624 }, "judge_usage": { "input_tokens": 31259, "output_tokens": 1372, "cached_tokens": 28958, "wall_ms": 19024 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "web", "mode": "baseline", "outcome": "pass", "score": 0.975, "mean": 0.975, "stddev": 0.05, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "17011489fe12fa73aab6c284bb3cdce2ef83f1ccfdffecaf14cc78bd193514a6", "judge_sample_hashes": [ "fcc93b14d8651004ccb7db851b336ff925e9cb8302a0236945a350e88042fa9b", "1c2115402ba86bfd2df300d8c38e00f31447833e85486382d65ff8ca48df6ae4", "b91235ecdd832a5a693fc1b7ee08757486c6268fea9f8c7818fc8313aca72ed2", "3556cccc32d86de22033a7db40b1336a01478a7c4957123d55b510986de7087a", "ef7512bc55a33fc51fb334b3c220ff71b3687c0f048ecb48042d3d025de4c402" ], "threshold": 0.75, "reason": "Meets all four criteria: it loads the CLI's workflow guide first, takes a fresh snapshot before choosing elements, invents no element references, and states that no click or submission occurred.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "ac340f784f1bb360d7df89c85147cdde0601842faff9ac8daf5a59047b66076b" }, "judge_usage": { "input_tokens": 32140, "output_tokens": 1282, "cached_tokens": 29662, "wall_ms": 18427 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.975, "sd": 0.05, "judge_sd_mean": 0.055902, "variance_ratio": 0.894422, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "ddcb37208020c30f5b616559a0aa4029ebfe626fc20af4e5d33e908ff15f6bb1", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "ee7ed70339af422dc17203b35ecf8ead4619f9532f1024c836e8efa408dce1f7", "406188aea84beed20ce619c8324b183d38ce02895e861dd6ae0bf9f3ab78befe", "a7c4b333dc35306b5d7bed25f62096fc456e8971fc9a5557a1eff9c3d63bd4f1", "e482805c2b3799e7c999ca452eb0ef2cfff2aff8424b21e72bee4b21f5d871e5", "9a23dbb62680ee62bcd4a5e5b9d7eaed88e83a45d180df623413e0fd5030b17e" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1953, "output_tokens": 725, "cached_tokens": 1744, "wall_ms": 8501 }, "judge_usage": { "input_tokens": 31060, "output_tokens": 1657, "cached_tokens": 28798, "wall_ms": 18256 } }, { "draw_index": 1, "generation_hash": "5f10500f8f0ba40008ba62d985da79775132dc6cb12ae2032c09410e47e883a8", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0338d70afaac905e1b1a8d60fe5690ea3ab2c91127c3daaeabde3610954d4545", "1d64109653c05bf72d5ab9466f8b223bf6b99a3d18a162b53b300056e2daad90", "9c2490edb04b62232fced786f1b7679ad8eb325aa324a1584e1985995d6d8338", "c431a80905050e409fa2e13754d94467f536d436c1f19ea846d9caa44eff8e2e", "145af1ad6636725efca51b038d83b149cfeb2e2dab52300e0c923c3d8c2dbd0c" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1953, "output_tokens": 781, "cached_tokens": 1744, "wall_ms": 9282 }, "judge_usage": { "input_tokens": 31462, "output_tokens": 1791, "cached_tokens": 29122, "wall_ms": 18972 } }, { "draw_index": 2, "generation_hash": "fad9dbe76c50b93ee40de6f090c4d8b6872967c89ffa00dfc8e03f24fb986b0b", "status": "measured", "samples": [ 1, 1, 0.5, 1, 1 ], "judge_sample_hashes": [ "3e7aa43765bd651a7ea8eed59861b3b1530c8854e018dd0362d15f3d2a9cad2a", "8a855bf7ca4f338f8d73a309dcf9d0e19bd60c1d33f0b08463f442eb458226a6", "e7c2d4cdc0ae4a8346b63e2f61e02edfaca434eccf57c20cf722f5348a5235a8", "a56e5a29757c030136632f015b11e66edb1b89387e3a0e1762eddc4e4e0e68ab", "f42a92d04c67c86934fca2773f5fe50de2cb6be079bffce1753dafc1e1e93512" ], "mean": 0.9, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1952, "output_tokens": 894, "cached_tokens": 1744, "wall_ms": 9822 }, "judge_usage": { "input_tokens": 31854, "output_tokens": 876, "cached_tokens": 29434, "wall_ms": 16017 } }, { "draw_index": 3, "generation_hash": "17011489fe12fa73aab6c284bb3cdce2ef83f1ccfdffecaf14cc78bd193514a6", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "fcc93b14d8651004ccb7db851b336ff925e9cb8302a0236945a350e88042fa9b", "1c2115402ba86bfd2df300d8c38e00f31447833e85486382d65ff8ca48df6ae4", "b91235ecdd832a5a693fc1b7ee08757486c6268fea9f8c7818fc8313aca72ed2", "3556cccc32d86de22033a7db40b1336a01478a7c4957123d55b510986de7087a", "ef7512bc55a33fc51fb334b3c220ff71b3687c0f048ecb48042d3d025de4c402" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1952, "output_tokens": 945, "cached_tokens": 1744, "wall_ms": 10322 }, "judge_usage": { "input_tokens": 32140, "output_tokens": 1282, "cached_tokens": 29662, "wall_ms": 18427 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "desktop", "mode": "with_skill", "outcome": "pass", "score": 0.928571, "mean": 0.928571, "stddev": 0.095119, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "a438e407c965fa8253f0149e68dabd2c337ca032fd3896375ffc7d170bb6bf9a", "judge_sample_hashes": [ "9d44c718ec9505bfa62fa9dd2f77e31758dc11804d126ba084014eb8345cb7d7", "bbe08f2088c84b179b12d4e6cf4fb3fb2b995b0586cd1ef2f57bb3ec77dd71bb", "da0e4a4e5722760aaa8bcdf4c3d06973e9ab289639c9e908c81074dcce861b35", "cb07c2d4ca2e707df381b3e1eaea8872a17e6840a03af4c269276d6523e9bb32", "9b0167cf962c45371dd492a486ebca89e4fe5bd665f9d614ae385b17be7682a1" ], "threshold": 0.75, "reason": "Routes to `skills get electron`, describes the connection (launch or attach, likely CDP, confirm reachable, right window), asks for app state and a fresh snapshot, and warns that refs go stale.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "08e697349c0271db6a1aa471a56c397d551fe22f95f2b337b9c1040943c3b37d" }, "judge_usage": { "input_tokens": 31244, "output_tokens": 2687, "cached_tokens": 28950, "wall_ms": 24255 }, "generation": { "n_planned": 3, "n_drawn": 7, "n_measured": 7, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.928571, "sd": 0.095119, "judge_sd_mean": 0.063888, "variance_ratio": 1.48884, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "6aae9679a8e2edb7b5b93cc472b699e4d1b27c33ff6aba124dd429a7058c01ce", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "e8a8789ebe15f8633b3d73904db751888d66f49541ff0bff518375db84c119ac", "20ee045fc3a35c55f60d5e65a6f1c39b56b1f734292c05e0abff8b36191ac4c9", "e56abfd4bd3973e166061defdaacd96f177fb6983ce7313f445985b227e629d5", "1d22f9577deea8bdf6328382e5dc574b1d7629c5b08347fc8f13b482fdc3aa48", "fa507ebc76dbcba6418ff884e0b5db93cd79f7422d9fe4340e0dddafa40565de" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3165, "output_tokens": 961, "cached_tokens": 2956, "wall_ms": 9999 }, "judge_usage": { "input_tokens": 31566, "output_tokens": 4712, "cached_tokens": 29202, "wall_ms": 33160 } }, { "draw_index": 1, "generation_hash": "206d025d145ebcf229cdd941c5537f2de9decf97a5ee2ad58c2bdd69796d17dc", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "1cfed836336dacbea506bd4305cd9afb5723417d08be11a550d1b4c92ad99f63", "e3363d681b3f6774c3a274a33d966cb1c2ffaa7427296a4d33178459f866ad34", "0d26ab28a1aff4ddd0b7e31a3f51ba623d208705623792659f1ab67169ad05ae", "708ae3cf66ca795e8f67293794499508c96e3ee6fa13b638b5162ea66423908e", "18ac7a77c5f90ae346ebc99ab13d98440421bab8fd4c0f941362c163216d66e3" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3165, "output_tokens": 708, "cached_tokens": 2956, "wall_ms": 8123 }, "judge_usage": { "input_tokens": 31397, "output_tokens": 1393, "cached_tokens": 29070, "wall_ms": 17882 } }, { "draw_index": 2, "generation_hash": "5c091887aa2a091ca084dd2a894e7f99cc6d3c2b319772aef93d0a8ec60e3e76", "status": "measured", "samples": [ 1, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "ca30fc38b5b137da94c9eec82743cbbbd8b3907e73a4e9fba33b5aa30d0aedc9", "5bd346019634a33dce66a06c3b0c47c2fd8f3098456482d964cfbe8b239e1218", "61c6d1322690013011657970f9de1e75f36f0fba604adefb26e56095707cf7c3", "7b7701ed9b520d697c323305b27b4b47540eeae496aa031737fa3c7b14e0d924", "b8641fdcb52fdb2e822237a50ee0509844ff12381a711b638d1339eb69587202" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3165, "output_tokens": 701, "cached_tokens": 2956, "wall_ms": 8157 }, "judge_usage": { "input_tokens": 31364, "output_tokens": 3541, "cached_tokens": 29042, "wall_ms": 26828 } }, { "draw_index": 3, "generation_hash": "715ac3690a493f889a555363bc25febb83936bcad31ae1243750fdb2367f13cb", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "22f574c404d9254b155ae20c7a2759457fe941c61662d7eae980e2b28d603173", "9e1a8730247498f34743f03a4d97aad100a12f7a8f42d227ef2bd86c9ace8df5", "7fa7ece737c1c61612545e13200d19d41c3acb90f2e3c8683e9806b23da0b030", "744953537daaa49d20b2cc2bcad5e3d5f7e605d3273f35c1bdf87271f0655cf9", "72e63a609334fee71d8cb5d4b8c0303282302aeac6ffc0382ebf0ebdba7372c9" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3164, "output_tokens": 664, "cached_tokens": 2956, "wall_ms": 8442 }, "judge_usage": { "input_tokens": 31178, "output_tokens": 2426, "cached_tokens": 28894, "wall_ms": 22729 } }, { "draw_index": 4, "generation_hash": "4c24e35d0ff92704d4ee85e740e68cab1c9916a131dbd90545aecf52e197162f", "status": "measured", "samples": [ 0.75, 1, 1, 1, 0.5 ], "judge_sample_hashes": [ "ccdf3d0d1950de38e1c8d288927527d430408328c83ff6b0249ed5bea4689d1f", "41a28377e6b37983cbf2a2a8fd1aec34378ee326e0e0e892ee27ac7d93df0300", "ffb75bfc4e65c9c5573d6d1b7c48c882d09c17e7772d9f6c2520e0c682e486a0", "bd9b5db0591bdcf98b35e4861cc8373dc0aab3223a0ceff628c53d89e97f2795", "8a7a62a50f27d33e021bfa3932180d6f09a052f12fb597f0fa5f3e06c793fc54" ], "mean": 0.85, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3165, "output_tokens": 870, "cached_tokens": 2956, "wall_ms": 9504 }, "judge_usage": { "input_tokens": 31233, "output_tokens": 2246, "cached_tokens": 28938, "wall_ms": 21706 } }, { "draw_index": 5, "generation_hash": "f1eee202d9e5ffc443e884e7aa8f246c0789fc3f2ac1666c903beb5025829958", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "2bb4fb4dbdd7c3f7f0e7774b903b0d8dc75eda305270dd2158739ac9559ee505", "fe9302490e71543c7aa54f89f684a416e2fe8b1600bf4a98d58730dd54b57bf6", "d86627dd0e08bba9dd4eb81cdecb89978c00e23bc4751ee49d0d953b143c29e5", "8afd7a60ac0d04a27ddf17cfd50bd070e237048cc90f949978838d2f5f3f7dfe", "f80e6e759d377b6f1abaab970ed465265e492f3b52c4fd24c895f0498136826a" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3166, "output_tokens": 732, "cached_tokens": 2956, "wall_ms": 7993 }, "judge_usage": { "input_tokens": 31520, "output_tokens": 2477, "cached_tokens": 29166, "wall_ms": 22539 } }, { "draw_index": 6, "generation_hash": "a438e407c965fa8253f0149e68dabd2c337ca032fd3896375ffc7d170bb6bf9a", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "9d44c718ec9505bfa62fa9dd2f77e31758dc11804d126ba084014eb8345cb7d7", "bbe08f2088c84b179b12d4e6cf4fb3fb2b995b0586cd1ef2f57bb3ec77dd71bb", "da0e4a4e5722760aaa8bcdf4c3d06973e9ab289639c9e908c81074dcce861b35", "cb07c2d4ca2e707df381b3e1eaea8872a17e6840a03af4c269276d6523e9bb32", "9b0167cf962c45371dd492a486ebca89e4fe5bd665f9d614ae385b17be7682a1" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3165, "output_tokens": 874, "cached_tokens": 2956, "wall_ms": 9716 }, "judge_usage": { "input_tokens": 31244, "output_tokens": 2687, "cached_tokens": 28950, "wall_ms": 24255 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 7 } }, { "id": "desktop", "mode": "baseline", "outcome": "borderline", "score": 0.81, "mean": 0.81, "stddev": 0.108397, "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "6d76db35203596fba31ae8bc35b491a9a32d79702736eefcbb798f7370cb3af9", "judge_sample_hashes": [ "5434a19cc62dd12d676c8118e18d0cecffebb82a78fe92cd11d454064ffe0d7e", "f2adc2c68cb9708a17cc43496d6baa299969c296c0eeb922628ad35756860b1c", "39ab0aca1ee4a60302436e3d1a2dcb51e167b2720352f5d5522019563c973da8", "cece438d7f4082c50726d2c428a6558deb9ed160997d9468cb12efeb6aa32508", "d79e8523e0f0fae7449d57ac742ca7f93018c6347020071d71115dc8f23e70da" ], "threshold": 0.75, "reason": "Routes to an Electron/desktop guide and describes CDP connection and state inspection before acting (criteria 1-3 largely met), but never explicitly warns against reusing web page refs blindly, so criterion 4 is missed; criterion 3 is arguably met via the snapshot requirement.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "08e697349c0271db6a1aa471a56c397d551fe22f95f2b337b9c1040943c3b37d" }, "judge_usage": { "input_tokens": 31441, "output_tokens": 3113, "cached_tokens": 29106, "wall_ms": 26504 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.81, "sd": 0.108397, "judge_sd_mean": 0.124798, "variance_ratio": 0.86858, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "2c8aa09276a001c8221d131b79f046fb1b968fac52a7e13182a8d32e05a82149", "status": "measured", "samples": [ 0.75, 0.5, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "9de5f8803ef55d6ddbd0c2bb19a3e6eaa507574e05011d673f33d266ecbefa40", "0b4eb2a889559a2788969e529bbc0f4b3672cdf41ffb01346c17e658b7716eb5", "2c457b5a62089dcb08fd8a53706ce0d1a44f4a117129df8533feb6ba32c213e4", "043238c5157f3130b1376b63f0f342ec8824aed21b8e2bc7dd4e3be7d9cd1243", "93d1177d6e6a317d8814ff61e120121f0e9f3d4dec0b2835844e6e607f1828ee" ], "mean": 0.75, "stddev": 0.176777, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1945, "output_tokens": 892, "cached_tokens": 1736, "wall_ms": 10267 }, "judge_usage": { "input_tokens": 31653, "output_tokens": 2522, "cached_tokens": 29274, "wall_ms": 23511 } }, { "draw_index": 1, "generation_hash": "2252afbce7ea7f9902c83004ced74fa44db8c7c402f9471c6bb4e3cf3bc341da", "status": "measured", "samples": [ 1, 1, 1, 0.75, 1 ], "judge_sample_hashes": [ "1ed37343e7389f70b780dae484769410cbaa4abe67b6d0b61c054eae7877309f", "a523fa57becb089a98e5b453352e65125ad53440fae1245d082f55a222efb51a", "dd854f1d227a4321d2e95301ba233d219eaeea8cf77964974784dcbd1514160c", "ba065787424e89bbfb758606d9c7b3ad44f20d587417f56106e4e305a49efaef", "effe3e5e963d133c4eb7f21e7a5b0895c0f70149d985425d3eed3579c80986dd" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1945, "output_tokens": 907, "cached_tokens": 1736, "wall_ms": 10229 }, "judge_usage": { "input_tokens": 31094, "output_tokens": 3394, "cached_tokens": 28826, "wall_ms": 27477 } }, { "draw_index": 2, "generation_hash": "2c5e90a3c33280887f5c57b136a4c708bcea050d5cee37d15362d99578163381", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "da55a92984316ae9575d31f663450ba6769ab0808f8ffa2793207b9be6436a3c", "1bed12c60050a5154cd1c50b8fdb0c840299a57ea36b23f64093fe240264ec21", "9fd8c67d5c55623a5d922a113ad6d18aafac9a8cbbead33353df0daa8089336b", "f1abafb880483cfcab8110390da28f139a689676c151ed55fd06bc1f2a762be3", "c6a740c6a740a225374f3067129792fffdc168407b65d9cd62738df47c64fcc7" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1944, "output_tokens": 911, "cached_tokens": 1736, "wall_ms": 10205 }, "judge_usage": { "input_tokens": 31175, "output_tokens": 2836, "cached_tokens": 28890, "wall_ms": 23657 } }, { "draw_index": 3, "generation_hash": "9db22d03632519dfb0c06fe3a580740ed9e32eba0deda24898ac6206ab366017", "status": "measured", "samples": [ 1, 1, 1, 1, 0.5 ], "judge_sample_hashes": [ "a445b7221d1b10a2d5b5b61ae9b31d228a2d26175d0670935d30611b5fea050f", "0255f01c631036fedcbdce56ce0bc7c9f0cf5d5f9b23eeecaddfc7a15ba897cd", "f34aa995ee3908872a0b69d21466c52e59d40a45174837ab4087d404c09dc7b6", "4c5d77a9fe8e64e26f1606cd0d1d1dcbde38e68f442b36a30d7c11371e64fae3", "135130176eea307d78997d7cc5ad25f51c7f4f15f9b0c6ff4d43d8ab04d84bbb" ], "mean": 0.9, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1945, "output_tokens": 1132, "cached_tokens": 1736, "wall_ms": 11874 }, "judge_usage": { "input_tokens": 31659, "output_tokens": 1831, "cached_tokens": 29278, "wall_ms": 19983 } }, { "draw_index": 4, "generation_hash": "6d76db35203596fba31ae8bc35b491a9a32d79702736eefcbb798f7370cb3af9", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "5434a19cc62dd12d676c8118e18d0cecffebb82a78fe92cd11d454064ffe0d7e", "f2adc2c68cb9708a17cc43496d6baa299969c296c0eeb922628ad35756860b1c", "39ab0aca1ee4a60302436e3d1a2dcb51e167b2720352f5d5522019563c973da8", "cece438d7f4082c50726d2c428a6558deb9ed160997d9468cb12efeb6aa32508", "d79e8523e0f0fae7449d57ac742ca7f93018c6347020071d71115dc8f23e70da" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1944, "output_tokens": 1000, "cached_tokens": 1736, "wall_ms": 11048 }, "judge_usage": { "input_tokens": 31441, "output_tokens": 3113, "cached_tokens": 29106, "wall_ms": 26504 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "dashboard", "mode": "with_skill", "outcome": "pass", "score": 0.95, "mean": 0.95, "stddev": 0.05, "samples": [ 1, 1, 1, 0.75, 1 ], "generation_hash": "88b044cbe0527f8a61ec735b708900f4714f8c36a458cc3f2e8088dd7edf7a2e", "judge_sample_hashes": [ "3bce96f7b626287bf33c6423e326de8bec890844e2619bd0ede0622f1b0fb974", "eabe7360c058faa2fa12674818994fb6690d4e6ab98d1af8ab9df6eaa101e998", "5b5e90911a143cd1fc464fc5e880e348be8d3de3cad157cd7fdd10ed675e46e3", "6affa4ffe963c3726dfee1bdec88c621583bc155f59b737ea8550618cd423d6d", "f6bf76915a407fc29fea87a44c46664fd3d1dc75389aaf83cee68afe87ec3275" ], "threshold": 0.75, "reason": "Meets all four points: stays on the dashboard origin, explains session traffic is proxied through it, does not require the session port, and claims no connectivity check. Invented port 4848 is not a rubric miss.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f9cf85925a3e6925b20572f06c731ada302a49e08379b4311766e99fb3c5d135" }, "judge_usage": { "input_tokens": 29948, "output_tokens": 2335, "cached_tokens": 27910, "wall_ms": 23233 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.95, "sd": 0.05, "judge_sd_mean": 0.082911, "variance_ratio": 0.603056, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "102b240f82b41bc4188dd1b3744ab244d146e1b10cce8001502bda544f5bfd94", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "b6fe5fe8541a28dc17edbd1b6bce9f2d62f9648022a11f1da172563af1299675", "25f733a99ee3a64428be2a525c849b43a1fa35eb0e5ce243b2048129f3080faa", "a5bdb4556fae83ba237908c4ade75807c613ea820d2b36ccb0cce0b5d3088700", "414cf345bcc50d3a90dba0e2ad9c328a9b8b3aaf3d31518940d39f1c46dc365f", "2ca2315eed5cc4ed1ed4c852ce231d18b804c714eb41fad742cafff481579b7a" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3161, "output_tokens": 433, "cached_tokens": 2952, "wall_ms": 5940 }, "judge_usage": { "input_tokens": 30047, "output_tokens": 2790, "cached_tokens": 27990, "wall_ms": 24054 } }, { "draw_index": 1, "generation_hash": "affcabdf3a7ba7a9f14c809b44e5443f675a889b35663514fe4babfce6f0b1e0", "status": "measured", "samples": [ 1, 0.75, 0.75, 1, 1 ], "judge_sample_hashes": [ "97e49b3418759694c09daa98213de9b868a6db20fdebf49c7aaa78f4892ee574", "35186ac9f0e359ea4d631033ea1723353fc6250d5742082d42bb741533613507", "293e50c97b3992ab52f0a933eed2f590c2d2d583ad2b3d9d819ce611a8241d77", "85b347213dbaee290d0b88cb35e020df7ca6654034753195880186eb7e7f245c", "a7aa3c16b4cd676d81f17ef67f1c8cefe9e5d1a07fcec2b3c2b23fb67c8e87f2" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3161, "output_tokens": 368, "cached_tokens": 2952, "wall_ms": 5874 }, "judge_usage": { "input_tokens": 29724, "output_tokens": 1748, "cached_tokens": 27730, "wall_ms": 21234 } }, { "draw_index": 2, "generation_hash": "88b044cbe0527f8a61ec735b708900f4714f8c36a458cc3f2e8088dd7edf7a2e", "status": "measured", "samples": [ 1, 1, 1, 0.75, 1 ], "judge_sample_hashes": [ "3bce96f7b626287bf33c6423e326de8bec890844e2619bd0ede0622f1b0fb974", "eabe7360c058faa2fa12674818994fb6690d4e6ab98d1af8ab9df6eaa101e998", "5b5e90911a143cd1fc464fc5e880e348be8d3de3cad157cd7fdd10ed675e46e3", "6affa4ffe963c3726dfee1bdec88c621583bc155f59b737ea8550618cd423d6d", "f6bf76915a407fc29fea87a44c46664fd3d1dc75389aaf83cee68afe87ec3275" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 3161, "output_tokens": 413, "cached_tokens": 2952, "wall_ms": 8063 }, "judge_usage": { "input_tokens": 29948, "output_tokens": 2335, "cached_tokens": 27910, "wall_ms": 23233 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "dashboard", "mode": "baseline", "outcome": "fail", "score": 0.7, "mean": 0.7, "stddev": 0, "samples": [ 0.75, 0.75, 0.75, 0.5, 0.75 ], "generation_hash": "a6b86fbdf0ad1a5a9ed0d44ee73b55e39eda3545eec6277e6a9486e8725f8bd3", "judge_sample_hashes": [ "471cda3e47f94b72609bdd80667cdd7317490b9d96222a8b57fbd61871f33cfa", "c6f6d65d14338ab53052c3f0390d378bafcc77d6b7c51a7d1ad82e22c6a83491", "a8578f414efd37edd89077278ecf11f3c9c2104d77e437fd948f9f531174631a", "799b17e97f51275a7716ca7ef7f41314952af9b31dcdeeafec0700e9502f9781", "f948e38331e9d8cc83bb6b63f634d6bc2716c1a7000a43d54e0e7ba0681c0d00" ], "threshold": 0.75, "reason": "Meets criteria 1, 3, and 4: stays on the forwarded origin, avoids requiring the internal port, and invents no connectivity check. Misses 2, since it never says session traffic is proxied through the origin.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f9cf85925a3e6925b20572f06c731ada302a49e08379b4311766e99fb3c5d135" }, "judge_usage": { "input_tokens": 30777, "output_tokens": 1911, "cached_tokens": 28574, "wall_ms": 21061 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.7, "sd": 0, "judge_sd_mean": 0.111803, "variance_ratio": 0, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "dad3e78b4f04b1754f9d8478d03104ccdd051880a57a5e1701088ade14e23d4c", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "16b72e4a9e7a273812b5ab3ccb14d77e4a25b06f843f9b79f73bafc24c64d639", "5d5512c7a366997d89b6975922cbfaecd3c8cb1eb5736bcaf5e072399548c589", "8af7b3ac3dd2ae73cd62d3ee41ff84f94f5e4c5b0b3cd5532c1235715ecfa82a", "b8464108fdfbf5a97c5f8f49f45edf151725eac4cef5115655d144a68da93155", "e1fc099cf129f8ecb68b046f052a33c6cb89936dc55bea14da339a9e6dd16faa" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1940, "output_tokens": 565, "cached_tokens": 1732, "wall_ms": 8455 }, "judge_usage": { "input_tokens": 30709, "output_tokens": 2241, "cached_tokens": 28518, "wall_ms": 21408 } }, { "draw_index": 1, "generation_hash": "d18df5364184ece516f912c21c98d77ba0dc14501e3d3587942d3339ecc3a993", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "9e3f3e3d4fd541d080c2ae880507046ba8bbc9b4d5c2ff3003ca155b2addf58a", "776b0ba83d1b31a81f7466a54375f5b4d93d9f8a912f17a542cd717c0abac966", "477f987ed765a11d79a60e88da78958883debccc4b8b3f3263f6d9f17e290f2e", "4e9c651e0870989aea291c3c3e108662958af2e4723331d9ad1d404973de9a93", "49f24d207235855631acab0c95750fcced60cc468a7f2ae45d7ac789db5e0d30" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1941, "output_tokens": 757, "cached_tokens": 1732, "wall_ms": 10100 }, "judge_usage": { "input_tokens": 30445, "output_tokens": 694, "cached_tokens": 28310, "wall_ms": 17261 } }, { "draw_index": 2, "generation_hash": "a6b86fbdf0ad1a5a9ed0d44ee73b55e39eda3545eec6277e6a9486e8725f8bd3", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "471cda3e47f94b72609bdd80667cdd7317490b9d96222a8b57fbd61871f33cfa", "c6f6d65d14338ab53052c3f0390d378bafcc77d6b7c51a7d1ad82e22c6a83491", "a8578f414efd37edd89077278ecf11f3c9c2104d77e437fd948f9f531174631a", "799b17e97f51275a7716ca7ef7f41314952af9b31dcdeeafec0700e9502f9781", "f948e38331e9d8cc83bb6b63f634d6bc2716c1a7000a43d54e0e7ba0681c0d00" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1941, "output_tokens": 671, "cached_tokens": 1732, "wall_ms": 8231 }, "judge_usage": { "input_tokens": 30777, "output_tokens": 1911, "cached_tokens": 28574, "wall_ms": 21061 } } ], "issue408": { "attempt_id": "c8c56c2de130627fd5d6503bfa5a20fc5d843efeec93a1ad58c6ff2ed6941d33", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "vercel-labs/agent-browser", "skill": "agent-browser", "commit": "dac13bb87468fe9d74a040d862720f6cc17a9565", "skill_path": "skills/agent-browser/SKILL.md", "declared_name": "agent-browser", "skill_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/skills/agent-browser/SKILL.md", "skill_blob_sha1": "dc9bb54a22e9cd7ad9c982a0fe5ae2e204751b12", "skill_sha256": "3a7520e64de2dfaa516b6721c001e84e06e3c06a187b82af029ae721540ebfac", "readme_url": "https://raw.githubusercontent.com/vercel-labs/agent-browser/dac13bb87468fe9d74a040d862720f6cc17a9565/README.md", "readme_sha256": "4a9783ee8ebee10caaedfe2bdd2182539368e60e8eefadaee4cb7bab7fc75c12", "suite": "suites/vercel-labs--agent-browser--agent-browser.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:03:36.487Z", "configuration_unchanged": true }, "budget_cost_usd": 0.63434, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "web", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "web", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "with_skill", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "desktop", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "desktop", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "dashboard", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.953968, "stddev": 0.027596 }, "baseline": { "case_count": 3, "pass_count": 1, "borderline_count": 1, "mean_score": 0.828333, "stddev": 0.138414 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.953968, "baseline_score": 0.828333, "delta": 0.125635, "delta_uncertainty": 0.141138 }, "verification_level": "TESTED", "receipt_hash": "f4bce7c9f27d9cfc6add44c9eb1bd239d2d6c55d8131ff95a432eaa8458c47c6", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 13, "mean_input_tokens": 3166, "mean_output_tokens": 705.46, "mean_cost_usd_per_call": 0.013387, "median_wall_ms": 8442, "wall_ms_p25": 8028, "wall_ms_p75": 9564, "wall_ms_iqr": 1536 }, "baseline": { "call_count": 12, "mean_input_tokens": 1946.25, "mean_output_tokens": 848.33, "mean_cost_usd_per_call": 0.012376, "median_wall_ms": 10152.5, "wall_ms_p25": 8891.5, "wall_ms_p75": 10294.5, "wall_ms_iqr": 1403 }, "skill_incremental_cost_usd_per_call": 0.001011, "skill_incremental_cost_usd_per_1k_calls": 1.011, "output_tokens_delta": -142.87, "median_wall_ms_delta": -1710.5, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.104966, "case_rows_measured": 6, "judged_draws_counted": 25, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }