{ "cells": [ { "skill": "code-review-and-quality", "A": "claude-opus-5", "B": "claude-opus-5-5", "columns": { "original": { "labels": [ "no separation detected" ], "cases": [ { "id": "severity-labeled-findings", "A": { "mean": 0.901, "sd": 0.007 }, "B": { "mean": 0.91, "sd": 0.015 }, "delta": 0.009, "label": "no separation detected" } ] }, "control": { "labels": [ "no separation detected" ], "cases": [ { "id": "severity-labeled-findings", "A": { "mean": 0.899, "sd": 0.005 }, "B": { "mean": 0.902, "sd": 0.016 }, "delta": 0.003, "label": "no separation detected" } ] }, "treatment": { "labels": [ "no separation detected" ], "cases": [ { "id": "severity-labeled-findings", "A": { "mean": 0.901, "sd": 0.013 }, "B": { "mean": 0.912, "sd": 0.012 }, "delta": 0.011, "label": "no separation detected" } ] } } }, { "skill": "git-workflow-and-versioning", "A": "claude-opus-5", "B": "claude-opus-5-5", "columns": { "original": { "labels": [ "no separation detected; Not enough draws to conclude at this effect floor" ], "cases": [ { "id": "commit-message-conventional-type", "A": { "mean": 0.861, "sd": 0.008 }, "B": { "mean": 0.838, "sd": 0.027 }, "delta": -0.023, "label": "no separation detected; Not enough draws to conclude at this effect floor" } ] }, "control": { "labels": [ "no separation detected" ], "cases": [ { "id": "commit-message-conventional-type", "A": { "mean": 0.853, "sd": 0.007 }, "B": { "mean": 0.844, "sd": 0.015 }, "delta": -0.009, "label": "no separation detected" } ] }, "treatment": { "labels": [ "no separation detected" ], "cases": [ { "id": "commit-message-conventional-type", "A": { "mean": 0.866, "sd": 0.005 }, "B": { "mean": 0.858, "sd": 0.01 }, "delta": -0.008, "label": "no separation detected" } ] } } }, { "skill": "documentation-and-adrs", "A": "claude-opus-5", "B": "claude-opus-5-5", "columns": { "original": { "labels": [ "no separation detected; Not enough draws to conclude at this effect floor" ], "cases": [ { "id": "comment-intent-not-implementation", "A": { "mean": 0.804, "sd": 0.103 }, "B": { "mean": 0.661, "sd": 0.148 }, "delta": -0.143, "label": "no separation detected; Not enough draws to conclude at this effect floor" } ] }, "control": { "labels": [ "no separation detected; Not enough draws to conclude at this effect floor" ], "cases": [ { "id": "comment-intent-not-implementation", "A": { "mean": 0.807, "sd": 0.109 }, "B": { "mean": 0.69, "sd": 0.147 }, "delta": -0.117, "label": "no separation detected; Not enough draws to conclude at this effect floor" } ] }, "treatment": { "labels": [ "no separation detected; Not enough draws to conclude at this effect floor" ], "cases": [ { "id": "comment-intent-not-implementation", "A": { "mean": 0.801, "sd": 0.099 }, "B": { "mean": 0.658, "sd": 0.105 }, "delta": -0.144, "label": "no separation detected; Not enough draws to conclude at this effect floor" } ] } } } ], "receipts": { "original": { "code-review-and-quality": { "claude-opus-5": { "with_skill": 0.901, "baseline": 0.862, "lift": 0.039, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.91, "baseline": 0.68, "lift": 0.23, "judge": "claude-opus-5", "level": "TESTED" } }, "git-workflow-and-versioning": { "claude-opus-5": { "with_skill": 0.861, "baseline": 0.765, "lift": 0.096, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.838, "baseline": 0.3, "lift": 0.538, "judge": "claude-opus-5", "level": "TESTED" } }, "documentation-and-adrs": { "claude-opus-5": { "with_skill": 0.804, "baseline": 0.567, "lift": 0.237, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.661, "baseline": 0.567, "lift": 0.094, "judge": "claude-opus-5", "level": "TESTED" } } }, "control": { "code-review-and-quality": { "claude-opus-5": { "with_skill": 0.899, "baseline": 0.87, "lift": 0.029, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.902, "baseline": 0.693, "lift": 0.209, "judge": "claude-opus-5", "level": "TESTED" } }, "git-workflow-and-versioning": { "claude-opus-5": { "with_skill": 0.853, "baseline": 0.766, "lift": 0.087, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.844, "baseline": 0.3, "lift": 0.544, "judge": "claude-opus-5", "level": "TESTED" } }, "documentation-and-adrs": { "claude-opus-5": { "with_skill": 0.807, "baseline": 0.583, "lift": 0.223, "judge": "claude-opus-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.69, "baseline": 0.573, "lift": 0.117, "judge": "claude-opus-5", "level": "TESTED" } } }, "treatment": { "code-review-and-quality": { "claude-opus-5": { "with_skill": 0.901, "baseline": 0.788, "lift": 0.113, "judge": "claude-opus-5-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.912, "baseline": 0.603, "lift": 0.309, "judge": "claude-opus-5-5", "level": "TESTED" } }, "git-workflow-and-versioning": { "claude-opus-5": { "with_skill": 0.866, "baseline": 0.774, "lift": 0.091, "judge": "claude-opus-5-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.858, "baseline": 0.3, "lift": 0.558, "judge": "claude-opus-5-5", "level": "TESTED" } }, "documentation-and-adrs": { "claude-opus-5": { "with_skill": 0.801, "baseline": 0.517, "lift": 0.285, "judge": "claude-opus-5-5", "level": "TESTED" }, "claude-opus-5-5": { "with_skill": 0.658, "baseline": 0.587, "lift": 0.071, "judge": "claude-opus-5-5", "level": "TESTED" } } } }, "agreement": { "treatment_vs_control": { "n": 51, "same_side": 50, "mean_abs_diff": 0.0291 }, "control_vs_original": { "n": 51, "same_side": 50, "mean_abs_diff": 0.0127 } } }