80 lines
3.0 KiB
Python
Executable File
80 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Evaluate the Needle sidecar against a reviewed golden set.
|
|
|
|
Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
|
|
|
|
Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
|
|
including "needs_review" for the fallback.
|
|
"""
|
|
import argparse, json, os, sys, time, urllib.request
|
|
from collections import Counter, defaultdict
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
|
|
def classify(url, text):
|
|
req = urllib.request.Request(
|
|
url.rstrip("/") + "/classify",
|
|
data=json.dumps({"text": text}).encode(),
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
return json.load(r)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
|
|
ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
|
|
ap.add_argument("--json", help="write per-row results here")
|
|
ap.add_argument("--min-conf", type=float, default=0.0,
|
|
help="predictions below this confidence count as needs_review")
|
|
a = ap.parse_args()
|
|
|
|
rows = [json.loads(l) for l in open(a.golden) if l.strip()]
|
|
results, confusion = [], Counter()
|
|
lat = []
|
|
for row in rows:
|
|
try:
|
|
out = classify(a.url, row["text"])
|
|
except Exception as e:
|
|
print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
|
|
sys.exit(2)
|
|
pred = out.get("action", "needs_review")
|
|
conf = out.get("confidence", 0.0)
|
|
if conf < a.min_conf:
|
|
pred = "needs_review"
|
|
lat.append(out.get("latency_ms", 0.0))
|
|
confusion[(row["expected"], pred)] += 1
|
|
results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
|
|
|
|
labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
|
|
print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
|
|
for l in labels:
|
|
tp = confusion[(l, l)]
|
|
pp = sum(v for (e, p), v in confusion.items() if p == l)
|
|
sup = sum(v for (e, p), v in confusion.items() if e == l)
|
|
prec = tp / pp if pp else 0.0
|
|
rec = tp / sup if sup else 0.0
|
|
print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
|
|
|
|
acc = sum(r["ok"] for r in results) / len(results)
|
|
print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
|
|
f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
|
|
|
|
for src in sorted({r.get("src", "?") for r in results}):
|
|
sub = [r for r in results if r.get("src", "?") == src]
|
|
print(f" {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")
|
|
|
|
wrong = [r for r in results if not r["ok"]]
|
|
if wrong:
|
|
print("\nmisses:")
|
|
for r in wrong:
|
|
print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
|
|
if a.json:
|
|
json.dump(results, open(a.json, "w"), indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|