more
This commit is contained in:
mats committed
1 parent
8f670dfa2f
commit
2237fbda55
15 files changed
+594
-52
No files matched your search
Binary file not shown.
@@ -0,0 +1,40 @@
|
||||
{"id": "ng-404-root", "src": "clickhouse", "expected": "log_404", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
|
||||
{"id": "ng-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
|
||||
{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
|
||||
{"id": "orch-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
|
||||
{"id": "orch-401-chat", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
|
||||
{"id": "ng-401-manifest", "src": "clickhouse", "expected": "auth_failure", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
|
||||
{"id": "ng-503-store", "src": "clickhouse", "expected": "escalate_outage", "note": "upstream 503, real user-facing route", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
|
||||
{"id": "ng-500-root", "src": "clickhouse", "expected": "escalate_outage", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
|
||||
{"id": "ng-500-wp", "src": "clickhouse", "expected": "security_alert", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
|
||||
{"id": "ng-500-blog", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
|
||||
{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
|
||||
{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
|
||||
{"id": "ng-404-env", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
|
||||
{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
|
||||
{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "security_alert", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
|
||||
{"id": "ng-404-commit", "src": "clickhouse", "expected": "log_404", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
|
||||
{"id": "ng-404-vite", "src": "clickhouse", "expected": "log_404", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
|
||||
{"id": "ng-404-blob", "src": "clickhouse", "expected": "log_404", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
|
||||
{"id": "ng-499-repos", "src": "clickhouse", "expected": "none", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
|
||||
{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
|
||||
{"id": "ng-308-blog", "src": "clickhouse", "expected": "none", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
|
||||
{"id": "ng-200-speed", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
|
||||
{"id": "ng-200-matrix", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
|
||||
{"id": "ng-200-overview", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
|
||||
{"id": "sched-readyz", "src": "clickhouse", "expected": "none", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
|
||||
{"id": "cart-ping", "src": "clickhouse", "expected": "none", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
|
||||
{"id": "rag-request", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
|
||||
{"id": "orch-session-load", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
|
||||
{"id": "worker-dialerr", "src": "clickhouse", "expected": "none", "note": "dependency down; arguably escalate_outage - decide", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
|
||||
{"id": "worker-eof", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
|
||||
{"id": "ledger-skip", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
|
||||
{"id": "web-cls", "src": "clickhouse", "expected": "record_vital", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
|
||||
{"id": "syn-js-1", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
|
||||
{"id": "syn-js-2", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
|
||||
{"id": "syn-vital-1", "src": "synthetic", "expected": "record_vital", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
|
||||
{"id": "syn-db-1", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
|
||||
{"id": "syn-db-2", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
|
||||
{"id": "syn-db-3", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
|
||||
{"id": "syn-oom", "src": "synthetic", "expected": "none", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
|
||||
{"id": "syn-retry", "src": "synthetic", "expected": "none", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate the Needle sidecar against a reviewed golden set.
|
||||
|
||||
Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
|
||||
|
||||
Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
|
||||
or "none" when no tool should fit. The sidecar has no fallback tool, so "none"
|
||||
rows measure how often it force-fits (see the confidence column).
|
||||
"""
|
||||
import argparse, json, os, sys, time, urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def classify(url, text):
|
||||
req = urllib.request.Request(
|
||||
url.rstrip("/") + "/classify",
|
||||
data=json.dumps({"text": text}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
|
||||
ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
|
||||
ap.add_argument("--json", help="write per-row results here")
|
||||
ap.add_argument("--min-conf", type=float, default=0.0,
|
||||
help="predictions below this confidence count as 'none'")
|
||||
a = ap.parse_args()
|
||||
|
||||
rows = [json.loads(l) for l in open(a.golden) if l.strip()]
|
||||
results, confusion = [], Counter()
|
||||
lat = []
|
||||
for row in rows:
|
||||
try:
|
||||
out = classify(a.url, row["text"])
|
||||
except Exception as e:
|
||||
print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
pred = out.get("action", "unclassified")
|
||||
conf = out.get("confidence", 0.0)
|
||||
if conf < a.min_conf or pred == "unclassified":
|
||||
pred = "none"
|
||||
lat.append(out.get("latency_ms", 0.0))
|
||||
confusion[(row["expected"], pred)] += 1
|
||||
results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
|
||||
|
||||
labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
|
||||
print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
|
||||
for l in labels:
|
||||
tp = confusion[(l, l)]
|
||||
pp = sum(v for (e, p), v in confusion.items() if p == l)
|
||||
sup = sum(v for (e, p), v in confusion.items() if e == l)
|
||||
prec = tp / pp if pp else 0.0
|
||||
rec = tp / sup if sup else 0.0
|
||||
print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
|
||||
|
||||
acc = sum(r["ok"] for r in results) / len(results)
|
||||
print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
|
||||
f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
|
||||
|
||||
wrong = [r for r in results if not r["ok"]]
|
||||
if wrong:
|
||||
print("\nmisses:")
|
||||
for r in wrong:
|
||||
print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
|
||||
if a.json:
|
||||
json.dump(results, open(a.json, "w"), indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -8,6 +8,8 @@ import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
import hashlib
|
||||
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
|
||||
import needle
|
||||
|
||||
@@ -50,6 +52,31 @@ def escalate_backend_outage(service: str, error_message: str):
|
||||
"""Escalate severe backend 5xx crash or unhandled panic to on-call."""
|
||||
return {"action": "escalate_outage", "service": service, "error": error_message}
|
||||
|
||||
@needle.tool
|
||||
def report_unclassified(service: str, summary: str):
|
||||
"""Fallback when the event matches none of the other categories: retries, cancellations, restarts, OOM, health checks, or anything unclear."""
|
||||
return {"action": "unclassified", "service": service, "summary": summary}
|
||||
|
||||
# --- Redaction: nothing PII-bearing should reach the model or the cache ---
|
||||
_REDACTIONS = [
|
||||
(re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), "<email>"),
|
||||
(re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "<ip>"),
|
||||
(re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), "<uuid>"),
|
||||
(re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), "<jwt>"),
|
||||
(re.compile(r"\b[0-9a-fA-F]{16,}\b"), "<hex>"),
|
||||
(re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1<v>"),
|
||||
]
|
||||
|
||||
def redact(text: str) -> str:
|
||||
for rx, sub in _REDACTIONS:
|
||||
text = rx.sub(sub, text)
|
||||
return text[:1000]
|
||||
|
||||
# Cache keyed on the redacted text: repeated signatures cost nothing.
|
||||
CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000))
|
||||
_cache = {}
|
||||
_stats = {"hits": 0, "misses": 0}
|
||||
|
||||
# Initialize Needle Agent once
|
||||
print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
|
||||
t0 = time.perf_counter()
|
||||
@@ -60,7 +87,8 @@ agent = needle.Needle(tools=[
|
||||
log_missing_resource_404,
|
||||
report_database_error,
|
||||
report_auth_failure,
|
||||
escalate_backend_outage
|
||||
escalate_backend_outage,
|
||||
report_unclassified
|
||||
])
|
||||
print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
|
||||
|
||||
@@ -86,6 +114,7 @@ class NeedleHandler(BaseHTTPRequestHandler):
|
||||
"status": "ok",
|
||||
"available": True,
|
||||
"model": "Cactus-Compute/needle3",
|
||||
"cache": {**_stats, "size": len(_cache)},
|
||||
"tools": [
|
||||
"client_js_crash",
|
||||
"security_alert",
|
||||
@@ -93,7 +122,8 @@ class NeedleHandler(BaseHTTPRequestHandler):
|
||||
"log_404",
|
||||
"database_error",
|
||||
"auth_failure",
|
||||
"escalate_outage"
|
||||
"escalate_outage",
|
||||
"unclassified"
|
||||
]
|
||||
}
|
||||
self.wfile.write(json.dumps(resp).encode("utf-8"))
|
||||
@@ -142,6 +172,20 @@ class NeedleHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
|
||||
return
|
||||
|
||||
text = redact(text)
|
||||
key = hashlib.sha256(text.encode()).hexdigest()
|
||||
cached = _cache.get(key)
|
||||
if cached is not None:
|
||||
_stats["hits"] += 1
|
||||
out = {**cached, "cached": True}
|
||||
self.send_response(200)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(out).encode("utf-8"))
|
||||
return
|
||||
_stats["misses"] += 1
|
||||
|
||||
t_start = time.perf_counter()
|
||||
agent.reset()
|
||||
res = agent.run(text, strict=False)
|
||||
@@ -166,6 +210,10 @@ class NeedleHandler(BaseHTTPRequestHandler):
|
||||
"model": "Cactus-Compute/needle3"
|
||||
}
|
||||
|
||||
if len(_cache) >= CACHE_MAX:
|
||||
_cache.pop(next(iter(_cache)))
|
||||
_cache[key] = out
|
||||
|
||||
self.send_response(200)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
|
||||
@@ -160,7 +160,9 @@ func createSchema(ctx context.Context, conn clickhouse.Conn, database string) er
|
||||
metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
|
||||
exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
|
||||
is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
|
||||
exact_signature String, pattern_signature String, version UInt8 DEFAULT 1
|
||||
exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
|
||||
method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
|
||||
origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
|
||||
) ENGINE = ReplacingMergeTree(version)
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (service, kind, pattern_signature, timestamp, event_id)
|
||||
@@ -175,6 +177,13 @@ func createSchema(ctx context.Context, conn clickhouse.Conn, database string) er
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS method String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS host String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS origin_host String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS referer_host String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_site String DEFAULT ''", database),
|
||||
fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_mode String DEFAULT ''", database),
|
||||
} {
|
||||
if err := conn.Exec(ctx, statement); err != nil {
|
||||
return err
|
||||
@@ -191,7 +200,7 @@ func appendEvent(batch driver.Batch, event normalization.Event) error {
|
||||
event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax,
|
||||
event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint,
|
||||
boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature,
|
||||
uint8(1),
|
||||
uint8(1), event.Method, event.Host, event.UserAgent, event.OriginHost, event.RefererHost, event.FetchSite, event.FetchMode,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -324,6 +324,7 @@ func main() {
|
||||
uiMux.Handle("/api/stats/errors", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsErrors)))
|
||||
uiMux.Handle("/api/stats/releases", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsReleases)))
|
||||
uiMux.Handle("/api/stats/services", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsServices)))
|
||||
uiMux.Handle("/api/stats/endpoints", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsEndpoints)))
|
||||
uiMux.HandleFunc("/ws", wsServer.HandleConnections)
|
||||
uiMux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static"))))
|
||||
// Serve the agent instructions doc as a static route so an AI agent can
|
||||
|
||||
Reference in new issue
Block a user