#!/usr/bin/env python3
"""D1 verdict: paired variant bootstrap of held-out success, last checkpoint minus step 0.

    python3 d1_analysis.py --step0 step0-heldout-k6.jsonl --last last-heldout-k6.jsonl [--seed 20261116]
        [--resamples 10000] [--min-valid-share 0.9] [--k 6] [--label heldout]

Records are the drums-verifiers DRUMS_RECORD lines (one per scored episode): fields used are `task` (the variant's task
id), `valid` (bool or null), `score` (the checker's score, 1.0 = success) and `ok`. Success = valid and score == 1.0.
Episodes that are not valid are not counted (they are not episodes, by the pre-registration); a variant's rate is its
successes over its valid episodes. The evaluation is complete when at least --min-valid-share of k x variants episodes are
valid and every variant has at least one valid episode. Verdict: percentile 95% CI of the mean over variants of
(last - step0), 10,000 resamples of variants with replacement, seed 20261116; met when the lower bound > 0.
Writes a JSON summary to stdout; exit code 0 always (the verdict is a field, not an exit status).
"""
from __future__ import annotations

import argparse
import json
import random
import sys
from collections import defaultdict


def load(path):
    per = defaultdict(lambda: {"valid": 0, "success": 0, "episodes": 0, "invalid": 0, "errors": 0})
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            r = json.loads(line)
            t = r.get("task") or r.get("task_id")
            if not t:
                continue
            p = per[t]
            p["episodes"] += 1
            if r.get("ok") is False and r.get("valid") is None:
                p["errors"] += 1
                continue
            if r.get("valid"):
                p["valid"] += 1
                if float(r.get("score") or 0.0) >= 1.0:
                    p["success"] += 1
            else:
                p["invalid"] += 1
    return per


def completeness(per, k, min_share):
    variants = len(per)
    valid = sum(p["valid"] for p in per.values())
    expected = k * variants
    every = all(p["valid"] >= 1 for p in per.values())
    return {"variants": variants, "valid_episodes": valid, "expected_episodes": expected,
            "valid_share": round(valid / expected, 4) if expected else 0.0,
            "every_variant_valid": every, "complete": bool(expected and valid / expected >= min_share and every)}


def rates(per):
    return {t: (p["success"] / p["valid"] if p["valid"] else None) for t, p in per.items()}


def paired_bootstrap(a, b, seed, n):
    """a, b: dict variant -> rate. Returns (mean diff, lo, hi, n_pairs) over variants with a rate in both."""
    keys = sorted(t for t in a if a[t] is not None and b.get(t) is not None)
    diffs = [b[t] - a[t] for t in keys]
    if not diffs:
        return None, None, None, 0
    rng = random.Random(seed)
    m = len(diffs)
    boots = []
    for _ in range(n):
        s = 0.0
        for _ in range(m):
            s += diffs[rng.randrange(m)]
        boots.append(s / m)
    boots.sort()
    lo = boots[int(0.025 * n)]
    hi = boots[min(n - 1, int(0.975 * n))]
    return sum(diffs) / m, lo, hi, m


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--step0", required=True)
    ap.add_argument("--last", required=True)
    ap.add_argument("--seed", type=int, default=20261116)
    ap.add_argument("--resamples", type=int, default=10000)
    ap.add_argument("--min-valid-share", type=float, default=0.9)
    ap.add_argument("--k", type=int, default=6)
    ap.add_argument("--label", default="heldout")
    a = ap.parse_args()
    p0, p1 = load(a.step0), load(a.last)
    c0, c1 = completeness(p0, a.k, a.min_valid_share), completeness(p1, a.k, a.min_valid_share)
    r0, r1 = rates(p0), rates(p1)
    mean0 = sum(v for v in r0.values() if v is not None) / max(1, sum(1 for v in r0.values() if v is not None))
    mean1 = sum(v for v in r1.values() if v is not None) / max(1, sum(1 for v in r1.values() if v is not None))
    d, lo, hi, m = paired_bootstrap(r0, r1, a.seed, a.resamples)
    met = bool(c0["complete"] and c1["complete"] and lo is not None and lo > 0)
    out = {
        "label": a.label, "seed": a.seed, "resamples": a.resamples,
        "step0": {"mean_rate": round(mean0, 4), **c0},
        "last": {"mean_rate": round(mean1, 4), **c1},
        "paired_variants": m,
        "diff": None if d is None else round(d, 4),
        "ci95": None if lo is None else [round(lo, 4), round(hi, 4)],
        "criterion": "lower bound > 0 with both evaluations complete",
        "met": met,
        "mixed_variants_step0": sum(1 for v in r0.values() if v is not None and 0 < v < 1),
        "floor_variants_step0": sum(1 for v in r0.values() if v == 0),
        "ceiling_variants_step0": sum(1 for v in r0.values() if v == 1),
    }
    json.dump(out, sys.stdout, indent=1)
    print()


if __name__ == "__main__":
    main()
