{
  "schema_version": "0.6",
  "generation_sampled": true,
  "skill": {
    "name": "documentation-and-adrs",
    "version": "0.0.0",
    "content_hash": "b67a9f07ed1057963d3d8ab04bb070ee74d93174eee48b563aa235057005c158",
    "tokens": 2454
  },
  "suite": {
    "format": "agentskills.io/evals",
    "suite_hash": "6cce54e7d9d0fc923eda5499c75fcc68974b3d8714b2a726e2ad1d77e4b5acc0",
    "case_count": 1,
    "canary": "86e8fbf6-9c40-ec04-dacc-03e730017455"
  },
  "run": {
    "model_id": "claude-haiku-4-5-20251001",
    "model_release_date": "2025-10-01",
    "provider": "anthropic",
    "surface": "claude-cli",
    "runner_version": "0.10.1",
    "date_utc": "2026-10-07T21:36:52.734Z",
    "registry": "registered",
    "transcripts": "retained-local",
    "judge": {
      "samples": 3,
      "temperature": null,
      "sampling": "surface-controlled",
      "surface": "claude-cli",
      "model_id": "claude-opus-5",
      "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed"
    },
    "answered_by": {
      "kind": "model",
      "attested": true,
      "reported_model": "claude-haiku-4-5-20251001",
      "reported_models": [
        "claude-haiku-4-5",
        "claude-opus-5"
      ],
      "isolation": "same-user"
    },
    "pricing_snapshot": {
      "frozen_at": "2026-10-07T21:36:52.734Z",
      "source": "config/models.json",
      "currency": "USD",
      "models": {
        "claude-haiku-4-5-20251001": {
          "input_per_mtok": 1,
          "output_per_mtok": 5,
          "registered": true
        },
        "claude-opus-5": {
          "input_per_mtok": 5,
          "output_per_mtok": 25,
          "registered": true
        }
      },
      "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change."
    }
  },
  "results": {
    "cases": [
      {
        "id": "comment-intent-not-implementation",
        "mode": "with_skill",
        "outcome": "pass",
        "score": 0.772,
        "mean": 0.772,
        "stddev": 0.06261,
        "samples": [
          0.8,
          0.8,
          0.8
        ],
        "generation_hash": "09d70baca0a6c50918d8825d8577ff68b21e64baac7c5d71d95cfb747903fbf4",
        "judge_sample_hashes": [
          "2bf9cfb183ebcee701c85c407f70aa99fb5137ed61b2c2fb92a86d340215f97c",
          "bad1f6ba29c89507618cb2e97f379c5f465a7ded8b02a0af9468453bf0dc6dde",
          "7ac49ddadfa15003a1e3166e5b228e4f0c1440d6edf8498aa5b783d2efe1b445"
        ],
        "threshold": 0.7,
        "reason": "Removes all three what-restating comments, the commented-out legacyRate line, and the bare TODO; no new restating comments, but adds no intent comment, so not exemplary.",
        "judge": {
          "model_id": "claude-opus-5",
          "rubric_hash": "b38f01e6671fc0b28a94de550fe70f879134a3168b35e5b1fb53a9a6773478b0"
        },
        "judge_usage": {
          "input_tokens": 44358,
          "output_tokens": 1253,
          "cached_tokens": 32827,
          "wall_ms": 34291
        },
        "generation": {
          "n_planned": 3,
          "n_drawn": 5,
          "n_measured": 5,
          "n_unmeasured": 0,
          "stopping_reason": "stabilised",
          "mean": 0.772,
          "sd": 0.06261,
          "judge_sd_mean": 0.010583,
          "variance_ratio": 5.916092,
          "variance_ratio_unavailable": null,
          "n_truncated": 0,
          "draws": [
            {
              "draw_index": 0,
              "generation_hash": "253e3ac5dafda9dea292c3ad095e9d21f397ba81f8aaf2261de40c4f2cc02ac5",
              "status": "measured",
              "samples": [
                0.7,
                0.6,
                0.68
              ],
              "judge_sample_hashes": [
                "a2872af36adb777674465474e08c02d9116cfce68b98d43d6b310105a8e31ea5",
                "f3514a7eb4d91e4d3314d0593e548527397cc51e00316da57804b634f080d2ea",
                "b72aa081c022364618804ceb8a793c47d7c163358b2d34bd72f967182d46de14"
              ],
              "mean": 0.66,
              "stddev": 0.052915,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 16600,
                "output_tokens": 1120,
                "cached_tokens": 16590,
                "wall_ms": 12810
              },
              "judge_usage": {
                "input_tokens": 44508,
                "output_tokens": 1544,
                "cached_tokens": 32927,
                "wall_ms": 49860
              }
            },
            {
              "draw_index": 1,
              "generation_hash": "50ee49f2aedd1d06a9e3454f8e3cd1b63c9a62af4dab7e65b04626146ce6ca66",
              "status": "measured",
              "samples": [
                0.8,
                0.8,
                0.8
              ],
              "judge_sample_hashes": [
                "273b5c362d9573b94680244a5a03273ea48ad5a0fb1b1819d9bb1499bcbc6f4d",
                "3dcb74862a5356c9841b109e28f216ce9713eb168b3c42dabf40f59893109b0b",
                "091b560db7fb314cd45c443308c8a3060be149ec8454ef8eefed118a86240cf0"
              ],
              "mean": 0.8,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 16600,
                "output_tokens": 773,
                "cached_tokens": 16590,
                "wall_ms": 9829
              },
              "judge_usage": {
                "input_tokens": 44628,
                "output_tokens": 935,
                "cached_tokens": 33007,
                "wall_ms": 20993
              }
            },
            {
              "draw_index": 2,
              "generation_hash": "89c34e785dd26f0f015c5f836bb03faf85e7738532e9784e75971bce6a71a97f",
              "status": "measured",
              "samples": [
                0.8,
                0.8,
                0.8
              ],
              "judge_sample_hashes": [
                "f30bd7e8b7c648e96de5352d6e3db64e50ca5186a9640d6a438b4f3c0dacad75",
                "273b5c362d9573b94680244a5a03273ea48ad5a0fb1b1819d9bb1499bcbc6f4d",
                "f7a30ad3d7ce1bb9bcf75e78aad1956932fa765cfa21ca4848c2be987c5d9c65"
              ],
              "mean": 0.8,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 16600,
                "output_tokens": 1970,
                "cached_tokens": 16590,
                "wall_ms": 20690
              },
              "judge_usage": {
                "input_tokens": 44457,
                "output_tokens": 1257,
                "cached_tokens": 32893,
                "wall_ms": 37579
              }
            },
            {
              "draw_index": 3,
              "generation_hash": "d6e549ea5cab552c28f558b98710ff9cf71c648da36a15e4f61690509a12e8d0",
              "status": "measured",
              "samples": [
                0.8,
                0.8,
                0.8
              ],
              "judge_sample_hashes": [
                "61d4017d3661468828c95248b5df63f7736291d34a5004abefe4d215917466ef",
                "09d5dd90556487a244c73a404478cf783c094ff4cae65f2d8242c6434de211c5",
                "30a0718d1bead996f15279e5568d71052b89858bef2abdddf4043eaf2ed21de6"
              ],
              "mean": 0.8,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 16600,
                "output_tokens": 787,
                "cached_tokens": 16590,
                "wall_ms": 9813
              },
              "judge_usage": {
                "input_tokens": 44352,
                "output_tokens": 1018,
                "cached_tokens": 32823,
                "wall_ms": 25476
              }
            },
            {
              "draw_index": 4,
              "generation_hash": "09d70baca0a6c50918d8825d8577ff68b21e64baac7c5d71d95cfb747903fbf4",
              "status": "measured",
              "samples": [
                0.8,
                0.8,
                0.8
              ],
              "judge_sample_hashes": [
                "2bf9cfb183ebcee701c85c407f70aa99fb5137ed61b2c2fb92a86d340215f97c",
                "bad1f6ba29c89507618cb2e97f379c5f465a7ded8b02a0af9468453bf0dc6dde",
                "7ac49ddadfa15003a1e3166e5b228e4f0c1440d6edf8498aa5b783d2efe1b445"
              ],
              "mean": 0.8,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 16600,
                "output_tokens": 2070,
                "cached_tokens": 16590,
                "wall_ms": 20628
              },
              "judge_usage": {
                "input_tokens": 44358,
                "output_tokens": 1253,
                "cached_tokens": 32827,
                "wall_ms": 34291
              }
            }
          ]
        }
      },
      {
        "id": "comment-intent-not-implementation",
        "mode": "baseline",
        "outcome": "fail",
        "score": 0.579167,
        "mean": 0.579167,
        "stddev": 0.053359,
        "samples": [
          0.6,
          0.6,
          0.6
        ],
        "generation_hash": "493a9ff72098ea1e7df3d4c32d64a90a07afd4ae0771fb8ea580ec0dae952a1d",
        "judge_sample_hashes": [
          "c5e7f45802748f6c6f0aab949e4f4aa806696470a69160d383fda6340ecd756b",
          "b13a4c1f6ec79eae5a938abb3b5990f7ffe9fe0990b8f8235337dc6db44dca57",
          "890e5c6d0a11ab9bed36e6ec4a949321b898a51a4acf004c69eac0c619eb81f2"
        ],
        "threshold": 0.7,
        "reason": "Met (a), (b), (d): removed all 'what'-restating comments and the commented-out legacyRate line; missed (c) by leaving the bare TODO dangling unaddressed.",
        "judge": {
          "model_id": "claude-opus-5",
          "rubric_hash": "b38f01e6671fc0b28a94de550fe70f879134a3168b35e5b1fb53a9a6773478b0"
        },
        "judge_usage": {
          "input_tokens": 44460,
          "output_tokens": 837,
          "cached_tokens": 32895,
          "wall_ms": 22303
        },
        "generation": {
          "n_planned": 3,
          "n_drawn": 4,
          "n_measured": 4,
          "n_unmeasured": 0,
          "stopping_reason": "stabilised",
          "mean": 0.579167,
          "sd": 0.053359,
          "judge_sd_mean": 0.019717,
          "variance_ratio": 2.706243,
          "variance_ratio_unavailable": null,
          "n_truncated": 0,
          "draws": [
            {
              "draw_index": 0,
              "generation_hash": "25e4a834c711f0e1cef7f21098425f011c1cd728493bda5b940e4dc02d57bd11",
              "status": "measured",
              "samples": [
                0.55,
                0.45,
                0.5
              ],
              "judge_sample_hashes": [
                "baa344e1886ba7e6ef8e01de8608f58e0d8566eb9b863f4dd099d045b27e35d2",
                "fb6862ed7850919e759992380245aae393759a0779d3dfe9d6118e61ff58382f",
                "0904b4f5e3675dfa768b86954c6191eeb39cdd1311d1f47ffea0abf68bbebe13"
              ],
              "mean": 0.5,
              "stddev": 0.05,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 14139,
                "output_tokens": 1497,
                "cached_tokens": 14129,
                "wall_ms": 15445
              },
              "judge_usage": {
                "input_tokens": 44547,
                "output_tokens": 1717,
                "cached_tokens": 32953,
                "wall_ms": 33077
              }
            },
            {
              "draw_index": 1,
              "generation_hash": "61d34db958b3df715cfb5350cb1b1ebda3ec0f1cfe75f2d90c7127df336a3289",
              "status": "measured",
              "samples": [
                0.6,
                0.6,
                0.65
              ],
              "judge_sample_hashes": [
                "9702096d1c8614708097b5ea620675d33c6ff33f0e7ea82156988e245620e94d",
                "6471c467e6c1f3a1f3117dee0c5ed2f29b95371e9adb555811d20f9a3a7a528d",
                "fe25b9a2fc440ded99d3ffb03cb6768e554e45e45caccf7796265e70dfe6366f"
              ],
              "mean": 0.616667,
              "stddev": 0.028868,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 14139,
                "output_tokens": 744,
                "cached_tokens": 14129,
                "wall_ms": 9655
              },
              "judge_usage": {
                "input_tokens": 44541,
                "output_tokens": 1279,
                "cached_tokens": 32949,
                "wall_ms": 26063
              }
            },
            {
              "draw_index": 2,
              "generation_hash": "150fc051356d1b9fc60cf7719cccfeaa701e39ecc2a62620e2af964f4e346aa1",
              "status": "measured",
              "samples": [
                0.6,
                0.6,
                0.6
              ],
              "judge_sample_hashes": [
                "31970a32fef839256743971715c5f861a5587e9208eb91a2ae4fcf965e3fec02",
                "20a513861dc376f1c64b8be00d396b9635a4d145181b3eb8b2c2f2160d80297e",
                "36e0b7861cbe2c7dacb0b1bb117aa055d4c64c57d42aface07f3571a09db8dc3"
              ],
              "mean": 0.6,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 14139,
                "output_tokens": 1078,
                "cached_tokens": 14129,
                "wall_ms": 12095
              },
              "judge_usage": {
                "input_tokens": 44427,
                "output_tokens": 1539,
                "cached_tokens": 32873,
                "wall_ms": 195094
              }
            },
            {
              "draw_index": 3,
              "generation_hash": "493a9ff72098ea1e7df3d4c32d64a90a07afd4ae0771fb8ea580ec0dae952a1d",
              "status": "measured",
              "samples": [
                0.6,
                0.6,
                0.6
              ],
              "judge_sample_hashes": [
                "c5e7f45802748f6c6f0aab949e4f4aa806696470a69160d383fda6340ecd756b",
                "b13a4c1f6ec79eae5a938abb3b5990f7ffe9fe0990b8f8235337dc6db44dca57",
                "890e5c6d0a11ab9bed36e6ec4a949321b898a51a4acf004c69eac0c619eb81f2"
              ],
              "mean": 0.6,
              "stddev": 0,
              "stop_reason": "end_turn",
              "truncated": false,
              "reported_model": "claude-haiku-4-5-20251001",
              "usage": {
                "input_tokens": 14139,
                "output_tokens": 1130,
                "cached_tokens": 14129,
                "wall_ms": 12353
              },
              "judge_usage": {
                "input_tokens": 44460,
                "output_tokens": 837,
                "cached_tokens": 32895,
                "wall_ms": 22303
              }
            }
          ]
        }
      }
    ],
    "aggregates": {
      "with_skill": {
        "case_count": 1,
        "pass_count": 1,
        "borderline_count": 0,
        "mean_score": 0.772,
        "stddev": null
      },
      "baseline": {
        "case_count": 1,
        "pass_count": 0,
        "borderline_count": 0,
        "mean_score": 0.579167,
        "stddev": null
      },
      "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases"
    }
  },
  "comparison": {
    "with_skill_score": 0.772,
    "baseline_score": 0.579167,
    "delta": 0.192833,
    "delta_uncertainty": null,
    "delta_uncertainty_unavailable": "single_case"
  },
  "verification_level": "TESTED",
  "receipt_hash": "3609ddc948814c96efd00ef98d89349bda96d29ca659265e784f82f478ccc44f",
  "economics": {
    "basis": "metered-equivalent",
    "surface": "claude-cli",
    "with_skill": {
      "call_count": 5,
      "mean_input_tokens": 16600,
      "mean_output_tokens": 1344,
      "mean_cost_usd_per_call": 0.02332,
      "median_wall_ms": 12810,
      "wall_ms_p25": 9821,
      "wall_ms_p75": 20659,
      "wall_ms_iqr": 10838
    },
    "baseline": {
      "call_count": 4,
      "mean_input_tokens": 14139,
      "mean_output_tokens": 1112.25,
      "mean_cost_usd_per_call": 0.0197,
      "median_wall_ms": 12224,
      "wall_ms_p25": 10875,
      "wall_ms_p75": 13899,
      "wall_ms_iqr": 3024
    },
    "skill_incremental_cost_usd_per_call": 0.00362,
    "skill_incremental_cost_usd_per_1k_calls": 3.62,
    "output_tokens_delta": 231.75,
    "median_wall_ms_delta": 586,
    "judge_excluded": true,
    "judge_overhead": {
      "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above.",
      "total_cost_usd": 0.49634,
      "case_rows_measured": 2
    },
    "notes": {
      "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.",
      "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.",
      "latency": "observed on subscription CLI surface, indicative"
    }
  }
}