#!/usr/bin/env python3
"""Evaluate the Needle sidecar against a reviewed golden set.

Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]

Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
including "needs_review" for the fallback.
"""
import argparse, json, os, sys, time, urllib.request
from collections import Counter, defaultdict

HERE = os.path.dirname(os.path.abspath(__file__))


def classify(url, text):
    req = urllib.request.Request(
        url.rstrip("/") + "/classify",
        data=json.dumps({"text": text}).encode(),
        headers={"Content-Type": "application/json"},
    )
    with urllib.request.urlopen(req, timeout=60) as r:
        return json.load(r)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
    ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
    ap.add_argument("--json", help="write per-row results here")
    ap.add_argument("--min-conf", type=float, default=0.0,
                    help="predictions below this confidence count as needs_review")
    a = ap.parse_args()

    rows = [json.loads(l) for l in open(a.golden) if l.strip()]
    results, confusion = [], Counter()
    lat = []
    for row in rows:
        try:
            out = classify(a.url, row["text"])
        except Exception as e:
            print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
            sys.exit(2)
        pred = out.get("action", "needs_review")
        conf = out.get("confidence", 0.0)
        if conf < a.min_conf:
            pred = "needs_review"
        lat.append(out.get("latency_ms", 0.0))
        confusion[(row["expected"], pred)] += 1
        results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})

    labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
    print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
    for l in labels:
        tp = confusion[(l, l)]
        pp = sum(v for (e, p), v in confusion.items() if p == l)
        sup = sum(v for (e, p), v in confusion.items() if e == l)
        prec = tp / pp if pp else 0.0
        rec = tp / sup if sup else 0.0
        print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")

    acc = sum(r["ok"] for r in results) / len(results)
    print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
          f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")

    for src in sorted({r.get("src", "?") for r in results}):
        sub = [r for r in results if r.get("src", "?") == src]
        print(f"  {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")

    wrong = [r for r in results if not r["ok"]]
    if wrong:
        print("\nmisses:")
        for r in wrong:
            print(f"  {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
    if a.json:
        json.dump(results, open(a.json, "w"), indent=1)


if __name__ == "__main__":
    main()