more
This commit is contained in:
mats committed
1 parent
8f670dfa2f
commit
2237fbda55
15 files changed
+594
-52
No files matched your search
@@ -0,0 +1,40 @@
|
||||
{"id": "ng-404-root", "src": "clickhouse", "expected": "log_404", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
|
||||
{"id": "ng-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
|
||||
{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
|
||||
{"id": "orch-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
|
||||
{"id": "orch-401-chat", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
|
||||
{"id": "ng-401-manifest", "src": "clickhouse", "expected": "auth_failure", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
|
||||
{"id": "ng-503-store", "src": "clickhouse", "expected": "escalate_outage", "note": "upstream 503, real user-facing route", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
|
||||
{"id": "ng-500-root", "src": "clickhouse", "expected": "escalate_outage", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
|
||||
{"id": "ng-500-wp", "src": "clickhouse", "expected": "security_alert", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
|
||||
{"id": "ng-500-blog", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
|
||||
{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
|
||||
{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
|
||||
{"id": "ng-404-env", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
|
||||
{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
|
||||
{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "security_alert", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
|
||||
{"id": "ng-404-commit", "src": "clickhouse", "expected": "log_404", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
|
||||
{"id": "ng-404-vite", "src": "clickhouse", "expected": "log_404", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
|
||||
{"id": "ng-404-blob", "src": "clickhouse", "expected": "log_404", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
|
||||
{"id": "ng-499-repos", "src": "clickhouse", "expected": "none", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
|
||||
{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
|
||||
{"id": "ng-308-blog", "src": "clickhouse", "expected": "none", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
|
||||
{"id": "ng-200-speed", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
|
||||
{"id": "ng-200-matrix", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
|
||||
{"id": "ng-200-overview", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
|
||||
{"id": "sched-readyz", "src": "clickhouse", "expected": "none", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
|
||||
{"id": "cart-ping", "src": "clickhouse", "expected": "none", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
|
||||
{"id": "rag-request", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
|
||||
{"id": "orch-session-load", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
|
||||
{"id": "worker-dialerr", "src": "clickhouse", "expected": "none", "note": "dependency down; arguably escalate_outage - decide", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
|
||||
{"id": "worker-eof", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
|
||||
{"id": "ledger-skip", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
|
||||
{"id": "web-cls", "src": "clickhouse", "expected": "record_vital", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
|
||||
{"id": "syn-js-1", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
|
||||
{"id": "syn-js-2", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
|
||||
{"id": "syn-vital-1", "src": "synthetic", "expected": "record_vital", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
|
||||
{"id": "syn-db-1", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
|
||||
{"id": "syn-db-2", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
|
||||
{"id": "syn-db-3", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
|
||||
{"id": "syn-oom", "src": "synthetic", "expected": "none", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
|
||||
{"id": "syn-retry", "src": "synthetic", "expected": "none", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate the Needle sidecar against a reviewed golden set.
|
||||
|
||||
Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
|
||||
|
||||
Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
|
||||
or "none" when no tool should fit. The sidecar has no fallback tool, so "none"
|
||||
rows measure how often it force-fits (see the confidence column).
|
||||
"""
|
||||
import argparse, json, os, sys, time, urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def classify(url, text):
|
||||
req = urllib.request.Request(
|
||||
url.rstrip("/") + "/classify",
|
||||
data=json.dumps({"text": text}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
|
||||
ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
|
||||
ap.add_argument("--json", help="write per-row results here")
|
||||
ap.add_argument("--min-conf", type=float, default=0.0,
|
||||
help="predictions below this confidence count as 'none'")
|
||||
a = ap.parse_args()
|
||||
|
||||
rows = [json.loads(l) for l in open(a.golden) if l.strip()]
|
||||
results, confusion = [], Counter()
|
||||
lat = []
|
||||
for row in rows:
|
||||
try:
|
||||
out = classify(a.url, row["text"])
|
||||
except Exception as e:
|
||||
print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
pred = out.get("action", "unclassified")
|
||||
conf = out.get("confidence", 0.0)
|
||||
if conf < a.min_conf or pred == "unclassified":
|
||||
pred = "none"
|
||||
lat.append(out.get("latency_ms", 0.0))
|
||||
confusion[(row["expected"], pred)] += 1
|
||||
results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
|
||||
|
||||
labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
|
||||
print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
|
||||
for l in labels:
|
||||
tp = confusion[(l, l)]
|
||||
pp = sum(v for (e, p), v in confusion.items() if p == l)
|
||||
sup = sum(v for (e, p), v in confusion.items() if e == l)
|
||||
prec = tp / pp if pp else 0.0
|
||||
rec = tp / sup if sup else 0.0
|
||||
print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
|
||||
|
||||
acc = sum(r["ok"] for r in results) / len(results)
|
||||
print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
|
||||
f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
|
||||
|
||||
wrong = [r for r in results if not r["ok"]]
|
||||
if wrong:
|
||||
print("\nmisses:")
|
||||
for r in wrong:
|
||||
print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
|
||||
if a.json:
|
||||
json.dump(results, open(a.json, "w"), indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user