Remove Needle; stop paging on nginx 4xx
Needle scored ~chance on the golden set and never gated alerts, so drop the sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled eval sets move to cmd/triage-eval/data. nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped in FilterActionable instead of only being discounted, so it cannot ride along with a real 5xx. nginx 5xx still alerts. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed
1 parent
e046bd86d1
commit
51a8412ae1
22 files changed
+29
-1162
No files matched your search
@@ -5,10 +5,6 @@ RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}"
|
||||
PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}"
|
||||
UPLOAD_SOURCEMAPS="${UPLOAD_SOURCEMAPS:-true}"
|
||||
|
||||
echo "Building Needle 3 sidecar image..."
|
||||
docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar
|
||||
docker push matst80/needle-sidecar:latest
|
||||
|
||||
echo "Building otel-debug receiver image..."
|
||||
docker build \
|
||||
--build-arg "APP_RELEASE=${RELEASE}" \
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
# Create non-root user with UID 1000 matching Kubernetes pod securityContext
|
||||
RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV HOME=/app
|
||||
ENV NEEDLE_PORT=5005
|
||||
ENV NEEDLE_HOST=0.0.0.0
|
||||
ENV NEEDLE_TELEMETRY=0
|
||||
|
||||
# Install dependencies
|
||||
RUN pip install --no-cache-dir cactus-needle
|
||||
|
||||
# Pre-cache model and shared library into /app/.cache as user 1000
|
||||
RUN python -c "import needle; needle.Needle()"
|
||||
|
||||
COPY server.py /app/server.py
|
||||
|
||||
# Ensure UID 1000 owns /app and all caches
|
||||
RUN chown -R appuser:appuser /app
|
||||
|
||||
USER appuser
|
||||
|
||||
EXPOSE 5005
|
||||
|
||||
CMD ["python", "/app/server.py"]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,42 +0,0 @@
|
||||
"""Deterministic baseline for the cause categories. Needle has to beat this."""
|
||||
import re
|
||||
|
||||
PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
|
||||
DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
|
||||
CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
|
||||
BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
|
||||
LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
|
||||
|
||||
|
||||
def fields(text):
|
||||
d = {}
|
||||
for part in text.split(" | "):
|
||||
k, _, v = part.partition(": ")
|
||||
d[k.lower()] = v
|
||||
return d
|
||||
|
||||
|
||||
def classify(text):
|
||||
f = fields(text)
|
||||
status = int(f.get("status") or 0)
|
||||
route = f.get("route", "")
|
||||
blob = " ".join([f.get("title", ""), f.get("message", ""), route])
|
||||
if PROBE.search(route):
|
||||
return "scanner_probe"
|
||||
if BUG.search(blob):
|
||||
return "service_defect"
|
||||
if CAP.search(blob):
|
||||
return "capacity_saturation"
|
||||
if DEP.search(blob):
|
||||
return "dependency_unreachable"
|
||||
if status == 401:
|
||||
return "client_credential_issue"
|
||||
if status in (502, 503, 504):
|
||||
return "dependency_unreachable"
|
||||
if status >= 500:
|
||||
return "service_defect"
|
||||
if status == 404:
|
||||
return "stale_reference"
|
||||
if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
|
||||
return "expected_lifecycle"
|
||||
return "needs_review"
|
||||
@@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate the Needle sidecar against a reviewed golden set.
|
||||
|
||||
Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
|
||||
|
||||
Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
|
||||
including "needs_review" for the fallback.
|
||||
"""
|
||||
import argparse, json, os, sys, time, urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def classify(url, text):
|
||||
req = urllib.request.Request(
|
||||
url.rstrip("/") + "/classify",
|
||||
data=json.dumps({"text": text}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
|
||||
ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
|
||||
ap.add_argument("--json", help="write per-row results here")
|
||||
ap.add_argument("--min-conf", type=float, default=0.0,
|
||||
help="predictions below this confidence count as needs_review")
|
||||
a = ap.parse_args()
|
||||
|
||||
rows = [json.loads(l) for l in open(a.golden) if l.strip()]
|
||||
results, confusion = [], Counter()
|
||||
lat = []
|
||||
for row in rows:
|
||||
try:
|
||||
out = classify(a.url, row["text"])
|
||||
except Exception as e:
|
||||
print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
pred = out.get("action", "needs_review")
|
||||
conf = out.get("confidence", 0.0)
|
||||
if conf < a.min_conf:
|
||||
pred = "needs_review"
|
||||
lat.append(out.get("latency_ms", 0.0))
|
||||
confusion[(row["expected"], pred)] += 1
|
||||
results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
|
||||
|
||||
labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
|
||||
print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
|
||||
for l in labels:
|
||||
tp = confusion[(l, l)]
|
||||
pp = sum(v for (e, p), v in confusion.items() if p == l)
|
||||
sup = sum(v for (e, p), v in confusion.items() if e == l)
|
||||
prec = tp / pp if pp else 0.0
|
||||
rec = tp / sup if sup else 0.0
|
||||
print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
|
||||
|
||||
acc = sum(r["ok"] for r in results) / len(results)
|
||||
print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
|
||||
f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
|
||||
|
||||
for src in sorted({r.get("src", "?") for r in results}):
|
||||
sub = [r for r in results if r.get("src", "?") == src]
|
||||
print(f" {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")
|
||||
|
||||
wrong = [r for r in results if not r["ok"]]
|
||||
if wrong:
|
||||
print("\nmisses:")
|
||||
for r in wrong:
|
||||
print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
|
||||
if a.json:
|
||||
json.dump(results, open(a.json, "w"), indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,240 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Needle 3 AI Telemetry Triage Sidecar Service
|
||||
Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
import hashlib
|
||||
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
|
||||
import needle
|
||||
|
||||
PORT = int(os.environ.get("NEEDLE_PORT", 5005))
|
||||
HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
|
||||
|
||||
# --- Cause-level triage tools ---
|
||||
# These deliberately do not restate what the event already carries (status
|
||||
# code, is_error, severity). Each one is a judgement about *why* it happened
|
||||
# or *who should care*, which is what an alert reader needs.
|
||||
@needle.tool
|
||||
def report_scanner_probe(service: str, path: str, status_code: int):
|
||||
"""Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
|
||||
return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
|
||||
|
||||
@needle.tool
|
||||
def report_dependency_unreachable(service: str, dependency: str, error_message: str):
|
||||
"""A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
|
||||
return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
|
||||
|
||||
@needle.tool
|
||||
def report_client_credential_issue(service: str, route: str, reason: str):
|
||||
"""A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
|
||||
return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
|
||||
|
||||
@needle.tool
|
||||
def report_stale_reference(service: str, path: str):
|
||||
"""Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
|
||||
return {"action": "stale_reference", "service": service, "path": path}
|
||||
|
||||
@needle.tool
|
||||
def report_service_defect(service: str, error_message: str):
|
||||
"""A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
|
||||
return {"action": "service_defect", "service": service, "error": error_message}
|
||||
|
||||
@needle.tool
|
||||
def report_capacity_saturation(service: str, resource: str, error_message: str):
|
||||
"""Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
|
||||
return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
|
||||
|
||||
@needle.tool
|
||||
def report_expected_lifecycle(service: str, what: str):
|
||||
"""Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
|
||||
return {"action": "expected_lifecycle", "service": service, "what": what}
|
||||
|
||||
@needle.tool
|
||||
def report_needs_review(service: str, summary: str):
|
||||
"""Fallback when none of the other categories clearly applies."""
|
||||
return {"action": "needs_review", "service": service, "summary": summary}
|
||||
|
||||
# --- Redaction: nothing PII-bearing should reach the model or the cache ---
|
||||
_REDACTIONS = [
|
||||
(re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), "<email>"),
|
||||
(re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "<ip>"),
|
||||
(re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), "<uuid>"),
|
||||
(re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), "<jwt>"),
|
||||
(re.compile(r"\b[0-9a-fA-F]{16,}\b"), "<hex>"),
|
||||
(re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1<v>"),
|
||||
]
|
||||
|
||||
def redact(text: str) -> str:
|
||||
for rx, sub in _REDACTIONS:
|
||||
text = rx.sub(sub, text)
|
||||
return text[:1000]
|
||||
|
||||
# Cache keyed on the redacted text: repeated signatures cost nothing.
|
||||
CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000))
|
||||
_cache = {}
|
||||
_stats = {"hits": 0, "misses": 0}
|
||||
|
||||
# Initialize Needle Agent once
|
||||
print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
|
||||
t0 = time.perf_counter()
|
||||
agent = needle.Needle(tools=[
|
||||
report_scanner_probe,
|
||||
report_dependency_unreachable,
|
||||
report_client_credential_issue,
|
||||
report_stale_reference,
|
||||
report_service_defect,
|
||||
report_capacity_saturation,
|
||||
report_expected_lifecycle,
|
||||
report_needs_review,
|
||||
])
|
||||
print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
|
||||
|
||||
|
||||
class NeedleHandler(BaseHTTPRequestHandler):
|
||||
def _send_cors(self):
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token")
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self.send_response(204)
|
||||
self._send_cors()
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health" or self.path == "/api/needle/status":
|
||||
self.send_response(200)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
resp = {
|
||||
"status": "ok",
|
||||
"available": True,
|
||||
"model": "Cactus-Compute/needle3",
|
||||
"cache": {**_stats, "size": len(_cache)},
|
||||
"tools": [
|
||||
"scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
|
||||
"service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
|
||||
]
|
||||
}
|
||||
self.wfile.write(json.dumps(resp).encode("utf-8"))
|
||||
else:
|
||||
self.send_response(404)
|
||||
self._send_cors()
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
if self.path in ("/classify", "/api/needle/classify"):
|
||||
content_len = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(content_len)
|
||||
try:
|
||||
data = json.loads(body) if body else {}
|
||||
except Exception as e:
|
||||
self.send_response(400)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8"))
|
||||
return
|
||||
|
||||
text = data.get("text", "").strip()
|
||||
# If text is not provided, build from structured fields:
|
||||
if not text:
|
||||
parts = []
|
||||
if data.get("service"):
|
||||
parts.append(f"Service: {data['service']}")
|
||||
if data.get("kind"):
|
||||
parts.append(f"Kind: {data['kind']}")
|
||||
if data.get("title"):
|
||||
parts.append(f"Title: {data['title']}")
|
||||
if data.get("message"):
|
||||
parts.append(f"Message: {data['message']}")
|
||||
if data.get("url"):
|
||||
parts.append(f"URL: {data['url']}")
|
||||
if data.get("status_code"):
|
||||
parts.append(f"Status: {data['status_code']}")
|
||||
text = " | ".join(parts)
|
||||
|
||||
if not text:
|
||||
self.send_response(400)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
|
||||
return
|
||||
|
||||
text = redact(text)
|
||||
key = hashlib.sha256(text.encode()).hexdigest()
|
||||
cached = _cache.get(key)
|
||||
if cached is not None:
|
||||
_stats["hits"] += 1
|
||||
out = {**cached, "cached": True}
|
||||
self.send_response(200)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(out).encode("utf-8"))
|
||||
return
|
||||
_stats["misses"] += 1
|
||||
|
||||
t_start = time.perf_counter()
|
||||
agent.reset()
|
||||
res = agent.run(text, strict=False)
|
||||
latency_ms = (time.perf_counter() - t_start) * 1000
|
||||
|
||||
results = res.get("results", [])
|
||||
first_result = results[0] if results and isinstance(results[0], dict) else {}
|
||||
action = first_result.get("action", "needs_review")
|
||||
|
||||
calls = res.get("function_calls", [])
|
||||
tool_name = calls[0].get("name") if calls else "triage_tool"
|
||||
|
||||
out = {
|
||||
"success": True,
|
||||
"input_text": text,
|
||||
"action": action,
|
||||
"tool": tool_name,
|
||||
"arguments": first_result,
|
||||
"confidence": float(res.get("confidence", 0.0)),
|
||||
"reasoning": res.get("reasoning", ""),
|
||||
"latency_ms": round(latency_ms, 2),
|
||||
"model": "Cactus-Compute/needle3"
|
||||
}
|
||||
|
||||
if len(_cache) >= CACHE_MAX:
|
||||
_cache.pop(next(iter(_cache)))
|
||||
_cache[key] = out
|
||||
|
||||
self.send_response(200)
|
||||
self._send_cors()
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(out).encode("utf-8"))
|
||||
else:
|
||||
self.send_response(404)
|
||||
self._send_cors()
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
# Concise logging
|
||||
print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}")
|
||||
|
||||
|
||||
def run():
|
||||
server = ThreadingHTTPServer((HOST, PORT), NeedleHandler)
|
||||
print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nShutting down needle sidecar...")
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -490,8 +490,6 @@ func main() {
|
||||
uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
|
||||
uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps)))
|
||||
uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks)))
|
||||
uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus)
|
||||
uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify)
|
||||
|
||||
// Protected API & WebSocket routes
|
||||
uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
// spike-alerts replays the deterministic spike rule over a bounded window of
|
||||
// normalized_events and, optionally, asks the Needle sidecar what kind of
|
||||
// failure each top pattern is. It only reads ClickHouse.
|
||||
// normalized_events. It only reads ClickHouse.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -36,7 +33,6 @@ func main() {
|
||||
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
|
||||
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
|
||||
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
|
||||
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
|
||||
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
|
||||
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
|
||||
flag.Parse()
|
||||
@@ -75,7 +71,6 @@ func main() {
|
||||
log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
|
||||
log.Printf("loaded %d error events", len(events))
|
||||
|
||||
cache := map[string]classification{}
|
||||
first := start.Add(time.Duration(*baselineWindows) * *window)
|
||||
suppress := features.NewSuppressor(*cooldown)
|
||||
muted := map[triage.Category]int{}
|
||||
@@ -87,24 +82,11 @@ func main() {
|
||||
}
|
||||
for _, spike := range suppress.Filter(detected) {
|
||||
fired++
|
||||
var cls []classification
|
||||
for _, p := range spike.Patterns {
|
||||
c := classification{}
|
||||
if *needle != "" {
|
||||
if hit, ok := cache[p.Signature]; ok {
|
||||
c = hit
|
||||
} else {
|
||||
c = classify(*needle, needleText(p.Example))
|
||||
cache[p.Signature] = c
|
||||
}
|
||||
}
|
||||
cls = append(cls, c)
|
||||
}
|
||||
if *asJSON {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
|
||||
continue
|
||||
}
|
||||
printSpike(spike, cls)
|
||||
printSpike(spike)
|
||||
}
|
||||
}
|
||||
for c, n := range muted {
|
||||
@@ -113,28 +95,7 @@ func main() {
|
||||
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
type classification struct {
|
||||
Action string `json:"action"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Cached bool `json:"cached"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func classify(base, text string) classification {
|
||||
body, _ := json.Marshal(map[string]string{"text": text})
|
||||
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return classification{Action: "?", Err: err.Error()}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var out classification
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return classification{Action: "?", Err: err.Error()}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func needleText(e features.Event) string {
|
||||
func eventText(e features.Event) string {
|
||||
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
|
||||
if e.SpanName != "" {
|
||||
parts = append(parts, "Title: "+e.SpanName)
|
||||
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
|
||||
return strings.Join(parts, " | ")
|
||||
}
|
||||
|
||||
func printSpike(s features.Spike, cls []classification) {
|
||||
func printSpike(s features.Spike) {
|
||||
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
|
||||
for i, p := range s.Patterns {
|
||||
for _, p := range s.Patterns {
|
||||
tag := "rising"
|
||||
if p.Novel {
|
||||
tag = "NEW"
|
||||
}
|
||||
what := string(alerts.TriageOf(p.Example).Category)
|
||||
if cls[i].Action != "" {
|
||||
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
|
||||
}
|
||||
clients := ""
|
||||
if p.Clients > 0 {
|
||||
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
|
||||
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
|
||||
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
||||
clients += " [" + p.Example.UAClass + "]"
|
||||
}
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File renamed without changes.
File renamed without changes.
+1
-1
@@ -1,7 +1,7 @@
|
||||
# Error detection model
|
||||
|
||||
> Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they
|
||||
> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins.
|
||||
> differ (ClickHouse is now a LoadBalancer; Needle is removed), STATUS.md wins.
|
||||
|
||||
## Current handoff (2026-09-18)
|
||||
|
||||
|
||||
+13
-22
@@ -10,7 +10,6 @@ unless stated. Greenfield, single user: radical changes are fine.
|
||||
patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded.
|
||||
- Credentials are in `.env` (gitignored, mode 600). Load with
|
||||
`set -a; . ./.env; set +a`. `clickhouse-client` is installed locally.
|
||||
- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly.
|
||||
|
||||
## What exists now
|
||||
|
||||
@@ -21,7 +20,7 @@ unless stated. Greenfield, single user: radical changes are fine.
|
||||
| 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. |
|
||||
| Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). |
|
||||
| Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. |
|
||||
| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. |
|
||||
| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories. |
|
||||
| Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. |
|
||||
| Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. |
|
||||
| Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. |
|
||||
@@ -35,14 +34,14 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
|
||||
|
||||
## Findings worth remembering
|
||||
|
||||
- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes).
|
||||
Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`),
|
||||
so it picks by position, not content. Confidence is uncorrelated with
|
||||
correctness. Decision: **Needle stays dormant.** Intended later use: a gate that
|
||||
decides whether a larger model should act on an issue (affects anyone? fixable by
|
||||
a smart model?). Do not use it for categorization.
|
||||
- **Needle was removed (2026-11-01).** It scored 10% on the 40-row golden set (about
|
||||
chance for 8 classes), picked by tool position rather than content, and its
|
||||
confidence was uncorrelated with correctness. Sidecar, proxy handlers, UI and k8s
|
||||
container are gone; the labelled sets live in `cmd/triage-eval/data/`.
|
||||
- **Edge 4xx never pages.** nginx 4xx (registry 401 challenges, `/oauth2/auth`,
|
||||
scanner 405s) is dropped in `FilterActionable`; nginx 5xx still alerts.
|
||||
- **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing
|
||||
the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl`
|
||||
the rows). A fresh 60-row holdout is in `cmd/triage-eval/data/holdout.jsonl`
|
||||
(from 2026-09-18, unseen); it still needs your `expected` labels. Known
|
||||
probe-like paths that still land in `stale_reference`: `/ssl/server.key`,
|
||||
`/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`,
|
||||
@@ -78,7 +77,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
|
||||
1. Deploy the collector so normalized rows stay current; verify with
|
||||
`SELECT max(timestamp) FROM otel.normalized_events`.
|
||||
2. Label `holdout.jsonl` (fill `expected`), run
|
||||
`go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix
|
||||
`go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl`, fix
|
||||
the rule families that miss. Then re-export a fresh holdout so scoring stays
|
||||
honest.
|
||||
3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the
|
||||
@@ -164,7 +163,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
|
||||
```bash
|
||||
set -a; . ./.env; set +a
|
||||
|
||||
# replay the spike rule over a range (Needle off)
|
||||
# replay the spike rule over a range
|
||||
go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12
|
||||
# add -show-muted to see probes / stale / lifecycle patterns too
|
||||
|
||||
@@ -172,12 +171,7 @@ go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00
|
||||
go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z
|
||||
|
||||
# score the rules against labelled rows
|
||||
go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl
|
||||
|
||||
# Needle eval (needs the sidecar running; venv was in scratch and is gone)
|
||||
pip install cactus-needle # python 3.12 recommended
|
||||
NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py
|
||||
python cmd/needle-sidecar/eval/run_eval.py
|
||||
go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl
|
||||
```
|
||||
|
||||
Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows
|
||||
@@ -187,14 +181,11 @@ through a `kubectl port-forward` hung and dropped.
|
||||
|
||||
Everything is uncommitted. Files changed by this work: `internal/{storage,
|
||||
normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts,
|
||||
triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`,
|
||||
triage-eval,bad-actors,normalized-backfill,server}`,
|
||||
`k8s/{clickhouse,deployment}.yaml`, `.gitignore`.
|
||||
|
||||
**Not from this work; check before committing:** `AGENTS.md`,
|
||||
`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**`
|
||||
(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files
|
||||
under `cmd/needle-sidecar` show as modified; they should be untracked and
|
||||
gitignored.
|
||||
`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` (including a deleted `GaugeMeter.tsx`).
|
||||
|
||||
## IP details (MaxMind)
|
||||
|
||||
|
||||
-36
@@ -141,45 +141,9 @@
|
||||
Clear Logs
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="data-section" style="border-left: 4px solid #10b981;">
|
||||
<h2 style="border-bottom-color: #10b981; color: #065f46;">Needle 3 AI Telemetry Triage</h2>
|
||||
<p style="font-size: 13px; color: #666; margin-top: 0;">Test on-device telemetry classification with Cactus-Compute/needle3:</p>
|
||||
<div style="display: flex; gap: 8px; margin-bottom: 10px;">
|
||||
<input id="needle-input" type="text" style="flex: 1; padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" value="User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1" />
|
||||
<button id="needle-btn" style="background: #10b981; color: white; border: none; padding: 8px 16px; border-radius: 4px; font-weight: bold; cursor: pointer;" onclick="classifyWithNeedle()">
|
||||
Classify with Needle 3
|
||||
</button>
|
||||
</div>
|
||||
<div id="needle-result" class="data-content" style="background: #ecfdf5; border: 1px solid #a7f3d0; min-height: 40px; color: #064e3b;">
|
||||
Click "Classify with Needle 3" to test the sidecar...
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function classifyWithNeedle() {
|
||||
const input = document.getElementById("needle-input").value;
|
||||
const resultDiv = document.getElementById("needle-result");
|
||||
const btn = document.getElementById("needle-btn");
|
||||
if (!input.trim()) return;
|
||||
btn.textContent = "Classifying...";
|
||||
btn.disabled = true;
|
||||
try {
|
||||
const resp = await fetch("/api/needle/classify", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: input }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
resultDiv.innerHTML = `<pre style="margin: 0; font-family: monospace;">${JSON.stringify(data, null, 2)}</pre>`;
|
||||
} catch (err) {
|
||||
resultDiv.textContent = "Error: " + err;
|
||||
} finally {
|
||||
btn.textContent = "Classify with Needle 3";
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
let ws;
|
||||
let tracesCount = 0;
|
||||
let metricsCount = 0;
|
||||
|
||||
@@ -11,6 +11,13 @@ func TriageOf(e features.Event) triage.Result {
|
||||
IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
|
||||
}
|
||||
|
||||
// edgeClientError reports a 4xx seen at the nginx edge: registry auth
|
||||
// challenges, /oauth2/auth subrequests, 405s from scanners. These are the
|
||||
// caller's problem and never page, even next to a real 5xx.
|
||||
func edgeClientError(e features.Event) bool {
|
||||
return e.Service == "nginx" && e.StatusCode >= 400 && e.StatusCode < 500
|
||||
}
|
||||
|
||||
// FilterActionable keeps only patterns whose category should page a human
|
||||
// (unless keepMuted), recomputing each spike's excess so a probe storm cannot
|
||||
// alert by itself. muted, when non-nil, counts what was dropped per category.
|
||||
@@ -21,7 +28,7 @@ func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.
|
||||
sp.Excess = 0
|
||||
for _, p := range sp.Patterns {
|
||||
c := TriageOf(p.Example).Category
|
||||
if !keepMuted && !c.Actionable() {
|
||||
if !keepMuted && (!c.Actionable() || edgeClientError(p.Example)) {
|
||||
if muted != nil {
|
||||
muted[c]++
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -1007,110 +1006,6 @@ func (c *Collector) HandleStatsEndpoints(w http.ResponseWriter, r *http.Request)
|
||||
})
|
||||
}
|
||||
|
||||
// needleAddr returns the configured or default address for the Needle 3 sidecar.
|
||||
func needleAddr() string {
|
||||
addr := os.Getenv("OTEL_NEEDLE_ADDR")
|
||||
if addr == "" {
|
||||
addr = "http://127.0.0.1:5005"
|
||||
}
|
||||
return strings.TrimRight(addr, "/")
|
||||
}
|
||||
|
||||
// HandleNeedleStatus checks if the Needle 3 sidecar is reachable and reports its health.
|
||||
func (c *Collector) HandleNeedleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
target := needleAddr() + "/health"
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
resp, err := client.Get(target)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"available": false,
|
||||
"error": fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
|
||||
})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// HandleNeedleClassify proxies telemetry triage requests to the Needle 3 sidecar.
|
||||
func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// If the payload specifies an issue_id and no text, look up the issue
|
||||
var reqMap map[string]interface{}
|
||||
if err := json.Unmarshal(body, &reqMap); err == nil {
|
||||
if issueID, ok := reqMap["issue_id"].(string); ok && issueID != "" {
|
||||
if issue, ok := c.GetIssue(issueID); ok {
|
||||
var promptParts []string
|
||||
promptParts = append(promptParts, fmt.Sprintf("Service: %s", issue.Service))
|
||||
promptParts = append(promptParts, fmt.Sprintf("Title: %s", issue.Title))
|
||||
promptParts = append(promptParts, fmt.Sprintf("Kind: %s", issue.Kind))
|
||||
if len(issue.Samples) > 0 {
|
||||
latest := issue.Samples[len(issue.Samples)-1]
|
||||
if latest.Message != "" {
|
||||
promptParts = append(promptParts, fmt.Sprintf("Message: %s", latest.Message))
|
||||
}
|
||||
if latest.URL != "" {
|
||||
promptParts = append(promptParts, fmt.Sprintf("URL: %s", latest.URL))
|
||||
}
|
||||
if latest.HTTPStatus > 0 {
|
||||
promptParts = append(promptParts, fmt.Sprintf("Status: %d", latest.HTTPStatus))
|
||||
}
|
||||
if latest.HTTPRoute != "" {
|
||||
promptParts = append(promptParts, fmt.Sprintf("Route: %s", latest.HTTPRoute))
|
||||
}
|
||||
}
|
||||
reqMap["text"] = strings.Join(promptParts, " | ")
|
||||
reqMap["service"] = issue.Service
|
||||
reqMap["title"] = issue.Title
|
||||
body, _ = json.Marshal(reqMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
target := needleAddr() + "/classify"
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
proxyReq, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
http.Error(w, "failed to create proxy request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
proxyReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := client.Do(proxyReq)
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": false,
|
||||
"error": fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
|
||||
})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// stampFrontendClient fills in client.address and user_agent.original on
|
||||
// browser-originated log records (source=frontend) from the ingest request,
|
||||
// since the browser cannot know its own public IP. Values the SDK already
|
||||
|
||||
@@ -173,8 +173,6 @@ spec:
|
||||
value: "15m"
|
||||
- name: OTEL_HONEYPOT_TTL
|
||||
value: "1h"
|
||||
- name: OTEL_NEEDLE_ADDR
|
||||
value: "http://127.0.0.1:5005"
|
||||
# ~80% of the memory limit so the GC works harder before an OOMKill.
|
||||
- name: GOMEMLIMIT
|
||||
value: "800MiB"
|
||||
@@ -202,40 +200,6 @@ spec:
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
- name: needle-sidecar
|
||||
image: matst80/needle-sidecar:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 5005
|
||||
name: needle-http
|
||||
env:
|
||||
- name: HOME
|
||||
value: "/app"
|
||||
- name: NEEDLE_PORT
|
||||
value: "5005"
|
||||
- name: NEEDLE_HOST
|
||||
value: "0.0.0.0"
|
||||
- name: NEEDLE_TELEMETRY
|
||||
value: "0"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: needle-http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: needle-http
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "4000m"
|
||||
volumes:
|
||||
- name: log-dir
|
||||
emptyDir: {}
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
ScrollText,
|
||||
ShieldBan,
|
||||
Bot,
|
||||
Sparkles,
|
||||
Loader2,
|
||||
X,
|
||||
} from "lucide-react"
|
||||
@@ -38,7 +37,6 @@ import { useSubRoute } from "../lib/routes"
|
||||
import { CopyRef } from "./common/CopyRef"
|
||||
import { IpButton } from "./common/IpButton"
|
||||
import { IssueOccurrences } from "./IssueOccurrences"
|
||||
import { NeedleTestModal } from "./NeedleTestModal"
|
||||
import { FilterBar, type FilterChip } from "./FilterBar"
|
||||
import { CRAWLER_CLASSES, UA_CLASSES, uaClassLabel } from "../lib/ua-class"
|
||||
import type { Issue, IssueSample } from "../lib/types"
|
||||
@@ -94,16 +92,6 @@ interface DetectorTestResult {
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface NeedleResult {
|
||||
action?: string
|
||||
tool?: string
|
||||
confidence?: number
|
||||
reasoning?: string
|
||||
latency_ms?: number
|
||||
arguments?: Record<string, unknown>
|
||||
error?: string
|
||||
}
|
||||
|
||||
const statusTabs: { value: StatusFilter; label: string }[] = [
|
||||
{ value: "all", label: "All" },
|
||||
{ value: "open", label: "Open" },
|
||||
@@ -315,26 +303,8 @@ const IssueCard = ({
|
||||
detectedIPByAddress: Map<string, DetectedIP>
|
||||
}) => {
|
||||
const [expanded, setExpanded] = useState(false)
|
||||
const [needleResult, setNeedleResult] = useState<NeedleResult | null>(null)
|
||||
const [needleLoading, setNeedleLoading] = useState(false)
|
||||
const { sources, loading: sourcesLoading, error: sourcesError } = useIssueSources(issue.id, expanded)
|
||||
|
||||
const handleNeedleTriage = async () => {
|
||||
setNeedleLoading(true)
|
||||
try {
|
||||
const response = await authFetch("/api/needle/classify", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ issue_id: issue.id }),
|
||||
})
|
||||
setNeedleResult(await response.json())
|
||||
} catch (err) {
|
||||
setNeedleResult({ error: err instanceof Error ? err.message : String(err) })
|
||||
} finally {
|
||||
setNeedleLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
const meta = kindMeta[issue.kind] ?? { label: issue.kind, Icon: Bug }
|
||||
const tone = issueSeverityTone(issue)
|
||||
const latestSample = issue.samples[issue.samples.length - 1]
|
||||
@@ -505,57 +475,6 @@ const IssueCard = ({
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleNeedleTriage()}
|
||||
disabled={needleLoading}
|
||||
title="Classify this issue using local Needle 3 model"
|
||||
className="inline-flex h-8 items-center gap-2 rounded-[9px] border border-ds-line-strong px-3 text-[12px] font-medium text-ds-text-2 hover:text-ds-text disabled:opacity-50"
|
||||
>
|
||||
{needleLoading ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" aria-hidden="true" />
|
||||
) : (
|
||||
<Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
)}
|
||||
{needleLoading ? "Needle 3 triaging…" : "Needle 3 triage"}
|
||||
</button>
|
||||
{needleResult && (
|
||||
<div className="space-y-2 rounded-[12px] border border-ds-line bg-ds-surface-2 p-3 text-[12px]">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 border-b border-ds-line pb-2">
|
||||
<span className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-[10.5px] font-bold uppercase tracking-wider text-ds-text-3">Needle 3 action</span>
|
||||
<span className="rounded-[6px] border border-ds-line-strong px-2 py-0.5 font-[family-name:var(--font-data)] text-ds-text">
|
||||
{needleResult.action || needleResult.tool || "Processed"}
|
||||
</span>
|
||||
</span>
|
||||
{needleResult.confidence !== undefined && (
|
||||
<span className="font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
|
||||
confidence{" "}
|
||||
<span className={needleResult.confidence >= 0.6 ? "text-ds-ok" : "text-ds-warn"}>
|
||||
{(needleResult.confidence * 100).toFixed(0)}%
|
||||
</span>
|
||||
{needleResult.latency_ms ? ` (${needleResult.latency_ms}ms)` : ""}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{needleResult.reasoning && <p className="italic text-ds-text-2">“{needleResult.reasoning}”</p>}
|
||||
{needleResult.arguments && Object.keys(needleResult.arguments).length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 font-[family-name:var(--font-data)] text-[11px]">
|
||||
{Object.entries(needleResult.arguments).map(([k, v]) => (
|
||||
<span key={k} className="rounded-[6px] border border-ds-line bg-ds-surface px-2 py-0.5 text-ds-text-2">
|
||||
<span className="text-ds-text-3">{k}:</span> {String(v)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{needleResult.error && (
|
||||
<p className="font-[family-name:var(--font-data)] text-[11px] text-ds-critical">{needleResult.error}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-baseline justify-between gap-3">
|
||||
<p className="text-[10.5px] font-bold uppercase tracking-widest text-ds-text-3">
|
||||
@@ -750,7 +669,6 @@ export const IssuesVisualizer = ({
|
||||
const [statusFilter, setStatusFilter] = useState<StatusFilter>("all")
|
||||
const [kindFilter, setKindFilter] = useState<KindFilter>("all")
|
||||
const [groupBy, setGroupBy] = useState<GroupBy>("none")
|
||||
const [needleModalOpen, setNeedleModalOpen] = useState(false)
|
||||
const [detectorTest, setDetectorTest] = useState<DetectorTestResult | null>(null)
|
||||
const [detectorTestLoading, setDetectorTestLoading] = useState(false)
|
||||
const [excludeSpamIPs, setExcludeSpamIPs] = useState(false)
|
||||
@@ -1119,15 +1037,6 @@ export const IssuesVisualizer = ({
|
||||
<Bot className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
{crawlersHidden ? "Crawlers hidden" : "Hide crawlers"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setNeedleModalOpen(true)}
|
||||
title="Open interactive Needle 3 telemetry classifier"
|
||||
className="inline-flex h-9 items-center gap-1.5 rounded-[9px] border border-ds-line-strong px-2.5 text-[12px] font-medium text-ds-text-2 hover:text-ds-text"
|
||||
>
|
||||
<Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
Test Needle 3
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void runDetectorTest()}
|
||||
@@ -1295,7 +1204,6 @@ export const IssuesVisualizer = ({
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<NeedleTestModal isOpen={needleModalOpen} onClose={() => setNeedleModalOpen(false)} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
import React, { useState } from "react"
|
||||
import { X, Sparkles, Loader2, Cpu, CheckCircle2, AlertTriangle } from "lucide-react"
|
||||
import { authFetch } from "../lib/auth"
|
||||
|
||||
interface NeedleTestModalProps {
|
||||
isOpen: boolean
|
||||
onClose: () => void
|
||||
}
|
||||
|
||||
interface NeedleResponse {
|
||||
success?: boolean
|
||||
action?: string
|
||||
tool?: string
|
||||
confidence?: number
|
||||
reasoning?: string
|
||||
latency_ms?: number
|
||||
arguments?: Record<string, unknown>
|
||||
error?: string
|
||||
}
|
||||
|
||||
const PRESETS = [
|
||||
{
|
||||
label: "Frontend RUM Crash",
|
||||
service: "eventsourced-ai-web",
|
||||
text: "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
|
||||
},
|
||||
{
|
||||
label: "Security CVE Probe",
|
||||
service: "nginx",
|
||||
text: "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
|
||||
},
|
||||
{
|
||||
label: "Web Vital Metric",
|
||||
service: "storefront-web",
|
||||
text: "Store RUM metric on service storefront-web: record TTFB of 84 ms",
|
||||
},
|
||||
{
|
||||
label: "Benign 404 Route",
|
||||
service: "git.k6n.net",
|
||||
text: "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
|
||||
},
|
||||
{
|
||||
label: "Database Pool Deadlock",
|
||||
service: "checkout-service",
|
||||
text: "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
|
||||
},
|
||||
]
|
||||
|
||||
export const NeedleTestModal: React.FC<NeedleTestModalProps> = ({ isOpen, onClose }) => {
|
||||
const [inputText, setInputText] = useState(PRESETS[0].text)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [result, setResult] = useState<NeedleResponse | null>(null)
|
||||
|
||||
if (!isOpen) return null
|
||||
|
||||
const handleClassify = async () => {
|
||||
if (!inputText.trim()) return
|
||||
setLoading(true)
|
||||
setResult(null)
|
||||
try {
|
||||
const resp = await authFetch("/api/needle/classify", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: inputText }),
|
||||
})
|
||||
const data = await resp.json()
|
||||
setResult(data)
|
||||
} catch (err) {
|
||||
setResult({ error: err instanceof Error ? err.message : String(err) })
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-ds-bg/80 p-4 animate-fade-in">
|
||||
<div className="relative flex flex-col w-full max-w-2xl rounded-[14px] border border-ds-ok/30 bg-ds-bg/95 p-6 max-h-[90vh] overflow-y-auto">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between border-b border-ds-line pb-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="flex h-10 w-10 items-center justify-center rounded-xl bg-ds-ok/15 border border-ds-ok/30 text-ds-ok">
|
||||
<Cpu className="h-5 w-5" />
|
||||
</div>
|
||||
<div>
|
||||
<h2 className="text-base font-bold text-ds-text flex items-center gap-2">
|
||||
Needle 3 AI Telemetry Classifier
|
||||
<span className="rounded-full bg-ds-ok/15 border border-ds-ok/30 px-2 py-0.5 text-[10px] font-mono text-ds-ok">
|
||||
Cactus-Compute
|
||||
</span>
|
||||
</h2>
|
||||
<p className="text-xs text-ds-text-2">
|
||||
On-device ~35MB foundation model executing real-time tool calling and extraction on local CPU.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="rounded-lg p-1.5 text-ds-text-2 hover:bg-ds-fill hover:text-ds-text transition-colors"
|
||||
>
|
||||
<X className="h-5 w-5" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Preset Selector */}
|
||||
<div className="mt-4">
|
||||
<label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
|
||||
Quick Telemetry Presets (from ClickHouse)
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-2 mt-2">
|
||||
{PRESETS.map((p) => (
|
||||
<button
|
||||
key={p.label}
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setInputText(p.text)
|
||||
setResult(null)
|
||||
}}
|
||||
className={`rounded-lg border px-2.5 py-1 text-[11px] font-medium transition-all ${
|
||||
inputText === p.text
|
||||
? "border-ds-ok bg-ds-ok/15 text-ds-ok"
|
||||
: "border-ds-line bg-ds-surface-2 text-ds-text-2 hover:border-ds-line-strong hover:bg-ds-fill"
|
||||
}`}
|
||||
>
|
||||
{p.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Input Textarea */}
|
||||
<div className="mt-4">
|
||||
<label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
|
||||
Telemetry Event / Log / Trace Summary
|
||||
</label>
|
||||
<textarea
|
||||
value={inputText}
|
||||
onChange={(e) => setInputText(e.target.value)}
|
||||
rows={3}
|
||||
className="mt-1.5 w-full rounded-xl border border-ds-line bg-ds-surface-2 p-3 font-mono text-xs text-ds-text placeholder:text-ds-text-3 focus:border-ds-ok/50 focus:outline-none focus:ring-1 focus:ring-ds-ok/40"
|
||||
placeholder="Type or paste telemetry event..."
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Classify Button */}
|
||||
<div className="mt-3 flex justify-end">
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClassify}
|
||||
disabled={loading || !inputText.trim()}
|
||||
className="inline-flex items-center gap-2 rounded-xl bg-ds-surface-2 px-4 py-2 text-xs font-bold text-ds-on-accent transition-all hover:opacity-95 active:scale-95 disabled:opacity-50"
|
||||
>
|
||||
{loading ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
Classifying on CPU...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Sparkles className="h-4 w-4" />
|
||||
Classify with Needle 3
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Results Area */}
|
||||
{result && (
|
||||
<div className="mt-5 rounded-xl border border-ds-ok/30 bg-ds-ok/15 p-4 space-y-3">
|
||||
<div className="flex items-center justify-between border-b border-ds-ok/20 pb-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{result.error ? (
|
||||
<AlertTriangle className="h-4 w-4 text-ds-critical" />
|
||||
) : (
|
||||
<CheckCircle2 className="h-4 w-4 text-ds-ok" />
|
||||
)}
|
||||
<span className="text-xs font-bold uppercase tracking-wider text-ds-text-2">
|
||||
{result.error ? "Error" : "Triage Result"}
|
||||
</span>
|
||||
{result.action && (
|
||||
<span className="rounded-lg bg-ds-ok/15 border border-ds-ok/40 px-2 py-0.5 font-mono text-xs font-bold text-ds-ok">
|
||||
{result.action}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{result.confidence !== undefined && (
|
||||
<div className="flex items-center gap-3 text-xs font-mono">
|
||||
<span className="text-ds-text-2">
|
||||
Confidence:{" "}
|
||||
<span className="font-bold text-ds-ok">
|
||||
{(result.confidence * 100).toFixed(0)}%
|
||||
</span>
|
||||
</span>
|
||||
{result.latency_ms !== undefined && (
|
||||
<span className="text-ds-text-3">({result.latency_ms}ms)</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{result.reasoning && (
|
||||
<p className="text-xs text-ds-text-2 italic">
|
||||
“{result.reasoning}”
|
||||
</p>
|
||||
)}
|
||||
|
||||
{result.arguments && Object.keys(result.arguments).length > 0 && (
|
||||
<div className="space-y-1.5 pt-1">
|
||||
<span className="text-[10px] font-bold uppercase tracking-wider text-ds-text-2">
|
||||
Extracted Typed Arguments:
|
||||
</span>
|
||||
<div className="grid grid-cols-2 gap-2 font-mono text-xs">
|
||||
{Object.entries(result.arguments).map(([key, value]) => (
|
||||
<div
|
||||
key={key}
|
||||
className="rounded-lg border border-ds-line bg-ds-surface-2 p-2 text-ds-text break-all"
|
||||
>
|
||||
<span className="text-ds-text-3">{key}: </span>
|
||||
<span className="text-ds-ok font-semibold">{String(value)}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{result.error && (
|
||||
<div className="rounded-lg bg-ds-critical-bg border border-ds-critical/20 p-3 text-xs text-ds-critical">
|
||||
{result.error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# Use pre-configured virtual environment or fall back to local .venv
|
||||
VENV_DIR="/home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv"
|
||||
if [ ! -d "${VENV_DIR}" ]; then
|
||||
VENV_DIR="${ROOT_DIR}/.venv"
|
||||
fi
|
||||
|
||||
if [ ! -f "${VENV_DIR}/bin/python" ]; then
|
||||
echo "Setting up virtual environment at ${VENV_DIR}..."
|
||||
python3 -m venv "${VENV_DIR}"
|
||||
"${VENV_DIR}/bin/pip" install --upgrade pip
|
||||
"${VENV_DIR}/bin/pip" install cactus-needle pydantic
|
||||
fi
|
||||
|
||||
echo "Running Needle 3 OTel Verification Suite..."
|
||||
"${VENV_DIR}/bin/python" "${SCRIPT_DIR}/test_needle_classifier.py"
|
||||
@@ -1,168 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test and Verification Suite for Cactus-Compute/needle3 with OpenTelemetry data.
|
||||
Tests:
|
||||
1. Action/Tool Dispatch Accuracy (Ground Truth Verification)
|
||||
2. Field Extraction & Grounding
|
||||
3. Confidence Calibrations (Act vs. Escalate)
|
||||
4. Semantic Vector Embeddings (Issue Clustering)
|
||||
5. Execution Performance & Latency (Tokens/sec)
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
import needle
|
||||
except ImportError:
|
||||
print("Error: cactus-needle not found. Run with the virtualenv python:")
|
||||
print(" /home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv/bin/python " + __file__)
|
||||
sys.exit(1)
|
||||
|
||||
# --- Define Triage Tools ---
|
||||
@needle.tool
|
||||
def report_client_javascript_crash(service: str, message: str, url: str):
|
||||
"""Report client-side JavaScript crash or error logged by browser RUM."""
|
||||
return {"action": "client_js_crash", "service": service, "message": message, "url": url}
|
||||
|
||||
@needle.tool
|
||||
def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
|
||||
"""Alert on malicious vulnerability scans or exploit crawlers."""
|
||||
return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
|
||||
|
||||
@needle.tool
|
||||
def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
|
||||
"""Record RUM performance metrics like TTFB or FCP."""
|
||||
return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
|
||||
|
||||
@needle.tool
|
||||
def log_missing_resource_404(path: str, host: str):
|
||||
"""Log benign 404 Not Found requests for missing pages or git commits."""
|
||||
return {"action": "log_404", "path": path, "host": host}
|
||||
|
||||
@needle.tool
|
||||
def escalate_backend_outage(service: str, error_message: str):
|
||||
"""Escalate severe backend database timeout or service crash to on-call."""
|
||||
return {"action": "escalate_outage", "service": service, "error": error_message}
|
||||
|
||||
|
||||
def run_verification():
|
||||
print("=" * 70)
|
||||
print(" NEEDLE 3 OPENTELEMETRY CLASSIFIER - VERIFICATION SUITE")
|
||||
print("=" * 70)
|
||||
|
||||
# 1. Initialize Agent
|
||||
print("\n[Step 1] Initializing Needle 3 Engine...")
|
||||
t0 = time.perf_counter()
|
||||
agent = needle.Needle(tools=[
|
||||
report_client_javascript_crash,
|
||||
alert_vulnerability_probe,
|
||||
record_frontend_web_vital,
|
||||
log_missing_resource_404,
|
||||
escalate_backend_outage
|
||||
])
|
||||
init_ms = (time.perf_counter() - t0) * 1000
|
||||
print(f"✓ Needle 3 Engine initialized in {init_ms:.1f}ms")
|
||||
|
||||
# 2. Ground-Truth Test Cases (from actual ClickHouse telemetry)
|
||||
test_cases = [
|
||||
{
|
||||
"name": "Frontend Browser RUM Crash",
|
||||
"input": "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
|
||||
"expected_tool": "report_client_javascript_crash",
|
||||
"expected_fields": ["service", "url"]
|
||||
},
|
||||
{
|
||||
"name": "Malicious CVE Exploit Probe",
|
||||
"input": "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
|
||||
"expected_tool": "alert_vulnerability_probe",
|
||||
"expected_fields": ["target", "status_code"]
|
||||
},
|
||||
{
|
||||
"name": "Web Vital RUM Metric",
|
||||
"input": "Store RUM metric on service storefront-web: record TTFB of 84 ms",
|
||||
"expected_tool": "record_frontend_web_vital",
|
||||
"expected_fields": ["service", "metric", "ms"]
|
||||
},
|
||||
{
|
||||
"name": "Nginx Benign 404 Route",
|
||||
"input": "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
|
||||
"expected_tool": "log_missing_resource_404",
|
||||
"expected_fields": ["path", "host"]
|
||||
},
|
||||
{
|
||||
"name": "Critical Database Outage",
|
||||
"input": "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
|
||||
"expected_tool": "escalate_backend_outage",
|
||||
"expected_fields": ["service"]
|
||||
}
|
||||
]
|
||||
|
||||
print("\n[Step 2] Testing Classification & Tool Dispatch Accuracy...")
|
||||
passed = 0
|
||||
total = len(test_cases)
|
||||
latencies = []
|
||||
|
||||
for i, tc in enumerate(test_cases, 1):
|
||||
agent.reset()
|
||||
t_start = time.perf_counter()
|
||||
res = agent.run(tc["input"], strict=False)
|
||||
dur_ms = (time.perf_counter() - t_start) * 1000
|
||||
latencies.append(dur_ms)
|
||||
|
||||
results = res.get("results", [])
|
||||
matched_action = results[0].get("action") if results and isinstance(results[0], dict) else None
|
||||
|
||||
# Tool name to action map
|
||||
expected_action = tc["expected_tool"].replace("report_", "").replace("alert_", "").replace("record_", "").replace("log_", "").replace("escalate_", "")
|
||||
# Check match
|
||||
is_match = False
|
||||
if results and any(tc["expected_tool"] in str(c) or matched_action for c in results):
|
||||
is_match = True
|
||||
|
||||
status_str = "PASS" if is_match else "FAIL"
|
||||
if is_match:
|
||||
passed += 1
|
||||
|
||||
print(f"\n Test {i}/{total}: {tc['name']} -> [{status_str}] ({dur_ms:.1f}ms)")
|
||||
print(f" Confidence: {res.get('confidence', 0.0):.3f}")
|
||||
print(f" Reasoning : {res.get('reasoning')}")
|
||||
print(f" Dispatched: {results}")
|
||||
|
||||
print("\n" + "-" * 70)
|
||||
print(f"Accuracy Score: {passed}/{total} ({passed/total*100:.0f}%) | Avg Latency: {sum(latencies)/len(latencies):.1f}ms")
|
||||
print("-" * 70)
|
||||
|
||||
# 3. Embedding & Semantic Search Verification
|
||||
print("\n[Step 3] Testing Semantic Vector Embeddings for Near-Match Clustering...")
|
||||
emb_e1 = agent.embed("database connection timeout after 3000ms")
|
||||
emb_e2 = agent.embed("postgres pool exhausted: connection timed out")
|
||||
emb_e3 = agent.embed("invalid JWT signature token expired")
|
||||
|
||||
def cosine_similarity(v1, v2):
|
||||
dot = sum(a * b for a, b in zip(v1, v2))
|
||||
m1 = math.sqrt(sum(a * a for a in v1))
|
||||
m2 = math.sqrt(sum(b * b for b in v2))
|
||||
return dot / (m1 * m2)
|
||||
|
||||
sim_similar = cosine_similarity(emb_e1, emb_e2)
|
||||
sim_different = cosine_similarity(emb_e1, emb_e3)
|
||||
|
||||
print(f" Vector Dimensions: {len(emb_e1)}")
|
||||
print(f" Similarity between 2 DB timeouts : {sim_similar:.4f}")
|
||||
print(f" Similarity between DB and Auth : {sim_different:.4f}")
|
||||
|
||||
if sim_similar > sim_different:
|
||||
print("✓ Semantic clustering validated: related issues score higher similarity.")
|
||||
else:
|
||||
print("✗ Warning: semantic separation failed.")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(" VERIFICATION COMPLETE")
|
||||
print("=" * 70)
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_verification()
|
||||
Reference in new issue
Block a user