more

cmd/needle-sidecar/eval/run_eval.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Evaluate the Needle sidecar against a reviewed golden set.
+
+Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
+
+Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
+or "none" when no tool should fit. The sidecar has no fallback tool, so "none"
+rows measure how often it force-fits (see the confidence column).
+"""
+import argparse, json, os, sys, time, urllib.request
+from collections import Counter, defaultdict
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+def classify(url, text):
+    req = urllib.request.Request(
+        url.rstrip("/") + "/classify",
+        data=json.dumps({"text": text}).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=60) as r:
+        return json.load(r)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
+    ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
+    ap.add_argument("--json", help="write per-row results here")
+    ap.add_argument("--min-conf", type=float, default=0.0,
+                    help="predictions below this confidence count as 'none'")
+    a = ap.parse_args()
+
+    rows = [json.loads(l) for l in open(a.golden) if l.strip()]
+    results, confusion = [], Counter()
+    lat = []
+    for row in rows:
+        try:
+            out = classify(a.url, row["text"])
+        except Exception as e:
+            print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
+            sys.exit(2)
+        pred = out.get("action", "unclassified")
+        conf = out.get("confidence", 0.0)
+        if conf < a.min_conf or pred == "unclassified":
+            pred = "none"
+        lat.append(out.get("latency_ms", 0.0))
+        confusion[(row["expected"], pred)] += 1
+        results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
+
+    labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
+    print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
+    for l in labels:
+        tp = confusion[(l, l)]
+        pp = sum(v for (e, p), v in confusion.items() if p == l)
+        sup = sum(v for (e, p), v in confusion.items() if e == l)
+        prec = tp / pp if pp else 0.0
+        rec = tp / sup if sup else 0.0
+        print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
+
+    acc = sum(r["ok"] for r in results) / len(results)
+    print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
+          f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
+
+    wrong = [r for r in results if not r["ok"]]
+    if wrong:
+        print("\nmisses:")
+        for r in wrong:
+            print(f"  {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
+    if a.json:
+        json.dump(results, open(a.json, "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main()