mats/go-otel
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Needle; stop paging on nginx 4xx

Browse Source 
Needle scored ~chance on the golden set and never gated alerts, so drop the
sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled
eval sets move to cmd/triage-eval/data.

nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped
in FilterActionable instead of only being discounted, so it cannot ride along
with a real 5xx. nginx 5xx still alerts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed 2026-11-01 11:40:53 +01:00
1 parent e046bd86d1
commit 51a8412ae1
22 files changed
+29 -1162
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/needle-sidecar/Dockerfile
-28
View File
@@ -1,28 +0,0 @@
FROM python:3.12-slim
# Create non-root user with UID 1000 matching Kubernetes pod securityContext
RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser
WORKDIR /app
ENV HOME=/app
ENV NEEDLE_PORT=5005
ENV NEEDLE_HOST=0.0.0.0
ENV NEEDLE_TELEMETRY=0
# Install dependencies
RUN pip install --no-cache-dir cactus-needle
# Pre-cache model and shared library into /app/.cache as user 1000
RUN python -c "import needle; needle.Needle()"
COPY server.py /app/server.py
# Ensure UID 1000 owns /app and all caches
RUN chown -R appuser:appuser /app
USER appuser
EXPOSE 5005
CMD ["python", "/app/server.py"]
cmd/needle-sidecar/__pycache__/server.cpython-314.pyc
BIN
View File
Binary file not shown.
cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc
BIN
View File
Binary file not shown.
cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc
BIN
View File
Binary file not shown.
cmd/needle-sidecar/eval/rules.py
-42
View File
@@ -1,42 +0,0 @@
"""Deterministic baseline for the cause categories. Needle has to beat this."""
import re
PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
def fields(text):
d = {}
for part in text.split(" | "):
k, _, v = part.partition(": ")
d[k.lower()] = v
return d
def classify(text):
f = fields(text)
status = int(f.get("status") or 0)
route = f.get("route", "")
blob = " ".join([f.get("title", ""), f.get("message", ""), route])
if PROBE.search(route):
return "scanner_probe"
if BUG.search(blob):
return "service_defect"
if CAP.search(blob):
return "capacity_saturation"
if DEP.search(blob):
return "dependency_unreachable"
if status == 401:
return "client_credential_issue"
if status in (502, 503, 504):
return "dependency_unreachable"
if status >= 500:
return "service_defect"
if status == 404:
return "stale_reference"
if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
return "expected_lifecycle"
return "needs_review"
cmd/needle-sidecar/eval/run_eval.py
-79
View File
@@ -1,79 +0,0 @@
#!/usr/bin/env python3
"""Evaluate the Needle sidecar against a reviewed golden set.
Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
including "needs_review" for the fallback.
"""
import argparse, json, os, sys, time, urllib.request
from collections import Counter, defaultdict
HERE = os.path.dirname(os.path.abspath(__file__))
def classify(url, text):
req = urllib.request.Request(
url.rstrip("/") + "/classify",
data=json.dumps({"text": text}).encode(),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=60) as r:
return json.load(r)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
ap.add_argument("--json", help="write per-row results here")
ap.add_argument("--min-conf", type=float, default=0.0,
help="predictions below this confidence count as needs_review")
a = ap.parse_args()
rows = [json.loads(l) for l in open(a.golden) if l.strip()]
results, confusion = [], Counter()
lat = []
for row in rows:
try:
out = classify(a.url, row["text"])
except Exception as e:
print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
sys.exit(2)
pred = out.get("action", "needs_review")
conf = out.get("confidence", 0.0)
if conf < a.min_conf:
pred = "needs_review"
lat.append(out.get("latency_ms", 0.0))
confusion[(row["expected"], pred)] += 1
results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
for l in labels:
tp = confusion[(l, l)]
pp = sum(v for (e, p), v in confusion.items() if p == l)
sup = sum(v for (e, p), v in confusion.items() if e == l)
prec = tp / pp if pp else 0.0
rec = tp / sup if sup else 0.0
print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
acc = sum(r["ok"] for r in results) / len(results)
print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
for src in sorted({r.get("src", "?") for r in results}):
sub = [r for r in results if r.get("src", "?") == src]
print(f" {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")
wrong = [r for r in results if not r["ok"]]
if wrong:
print("\nmisses:")
for r in wrong:
print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
if a.json:
json.dump(results, open(a.json, "w"), indent=1)
if __name__ == "__main__":
main()
cmd/needle-sidecar/server.py
-240
View File
@@ -1,240 +0,0 @@
#!/usr/bin/env python3
"""
Needle 3 AI Telemetry Triage Sidecar Service
Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights.
"""
import os
import sys
import json
import time
import re
import hashlib
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
import needle
PORT = int(os.environ.get("NEEDLE_PORT", 5005))
HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
# --- Cause-level triage tools ---
# These deliberately do not restate what the event already carries (status
# code, is_error, severity). Each one is a judgement about *why* it happened
# or *who should care*, which is what an alert reader needs.
@needle.tool
def report_scanner_probe(service: str, path: str, status_code: int):
"""Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
@needle.tool
def report_dependency_unreachable(service: str, dependency: str, error_message: str):
"""A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
@needle.tool
def report_client_credential_issue(service: str, route: str, reason: str):
"""A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
@needle.tool
def report_stale_reference(service: str, path: str):
"""Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
return {"action": "stale_reference", "service": service, "path": path}
@needle.tool
def report_service_defect(service: str, error_message: str):
"""A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
return {"action": "service_defect", "service": service, "error": error_message}
@needle.tool
def report_capacity_saturation(service: str, resource: str, error_message: str):
"""Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
@needle.tool
def report_expected_lifecycle(service: str, what: str):
"""Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
return {"action": "expected_lifecycle", "service": service, "what": what}
@needle.tool
def report_needs_review(service: str, summary: str):
"""Fallback when none of the other categories clearly applies."""
return {"action": "needs_review", "service": service, "summary": summary}
# --- Redaction: nothing PII-bearing should reach the model or the cache ---
_REDACTIONS = [
(re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), "<email>"),
(re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "<ip>"),
(re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), "<uuid>"),
(re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), "<jwt>"),
(re.compile(r"\b[0-9a-fA-F]{16,}\b"), "<hex>"),
(re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1<v>"),
]
def redact(text: str) -> str:
for rx, sub in _REDACTIONS:
text = rx.sub(sub, text)
return text[:1000]
# Cache keyed on the redacted text: repeated signatures cost nothing.
CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000))
_cache = {}
_stats = {"hits": 0, "misses": 0}
# Initialize Needle Agent once
print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
t0 = time.perf_counter()
agent = needle.Needle(tools=[
report_scanner_probe,
report_dependency_unreachable,
report_client_credential_issue,
report_stale_reference,
report_service_defect,
report_capacity_saturation,
report_expected_lifecycle,
report_needs_review,
])
print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
class NeedleHandler(BaseHTTPRequestHandler):
def _send_cors(self):
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token")
def do_OPTIONS(self):
self.send_response(204)
self._send_cors()
self.end_headers()
def do_GET(self):
if self.path == "/health" or self.path == "/api/needle/status":
self.send_response(200)
self._send_cors()
self.send_header("Content-Type", "application/json")
self.end_headers()
resp = {
"status": "ok",
"available": True,
"model": "Cactus-Compute/needle3",
"cache": {**_stats, "size": len(_cache)},
"tools": [
"scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
"service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
]
}
self.wfile.write(json.dumps(resp).encode("utf-8"))
else:
self.send_response(404)
self._send_cors()
self.end_headers()
def do_POST(self):
if self.path in ("/classify", "/api/needle/classify"):
content_len = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(content_len)
try:
data = json.loads(body) if body else {}
except Exception as e:
self.send_response(400)
self._send_cors()
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8"))
return
text = data.get("text", "").strip()
# If text is not provided, build from structured fields:
if not text:
parts = []
if data.get("service"):
parts.append(f"Service: {data['service']}")
if data.get("kind"):
parts.append(f"Kind: {data['kind']}")
if data.get("title"):
parts.append(f"Title: {data['title']}")
if data.get("message"):
parts.append(f"Message: {data['message']}")
if data.get("url"):
parts.append(f"URL: {data['url']}")
if data.get("status_code"):
parts.append(f"Status: {data['status_code']}")
text = " | ".join(parts)
if not text:
self.send_response(400)
self._send_cors()
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
return
text = redact(text)
key = hashlib.sha256(text.encode()).hexdigest()
cached = _cache.get(key)
if cached is not None:
_stats["hits"] += 1
out = {**cached, "cached": True}
self.send_response(200)
self._send_cors()
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(out).encode("utf-8"))
return
_stats["misses"] += 1
t_start = time.perf_counter()
agent.reset()
res = agent.run(text, strict=False)
latency_ms = (time.perf_counter() - t_start) * 1000
results = res.get("results", [])
first_result = results[0] if results and isinstance(results[0], dict) else {}
action = first_result.get("action", "needs_review")
calls = res.get("function_calls", [])
tool_name = calls[0].get("name") if calls else "triage_tool"
out = {
"success": True,
"input_text": text,
"action": action,
"tool": tool_name,
"arguments": first_result,
"confidence": float(res.get("confidence", 0.0)),
"reasoning": res.get("reasoning", ""),
"latency_ms": round(latency_ms, 2),
"model": "Cactus-Compute/needle3"
}
if len(_cache) >= CACHE_MAX:
_cache.pop(next(iter(_cache)))
_cache[key] = out
self.send_response(200)
self._send_cors()
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(out).encode("utf-8"))
else:
self.send_response(404)
self._send_cors()
self.end_headers()
def log_message(self, format, *args):
# Concise logging
print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}")
def run():
server = ThreadingHTTPServer((HOST, PORT), NeedleHandler)
print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}")
try:
server.serve_forever()
except KeyboardInterrupt:
print("\nShutting down needle sidecar...")
server.server_close()
if __name__ == "__main__":
run()
cmd/server/main.go
-2
View File
@@ -490,8 +490,6 @@ func main() {
uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps)))
uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks)))
uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus)
uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify)
// Protected API & WebSocket routes
uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))
cmd/spike-alerts/main.go
+7 -49
View File
@@ -1,16 +1,13 @@
// spike-alerts replays the deterministic spike rule over a bounded window of
// normalized_events and, optionally, asks the Needle sidecar what kind of
// failure each top pattern is. It only reads ClickHouse.
// normalized_events. It only reads ClickHouse.
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"strings"
"time"
@@ -36,7 +33,6 @@ func main() {
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
flag.Parse()
@@ -75,7 +71,6 @@ func main() {
log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
log.Printf("loaded %d error events", len(events))
cache := map[string]classification{}
first := start.Add(time.Duration(*baselineWindows) * *window)
suppress := features.NewSuppressor(*cooldown)
muted := map[triage.Category]int{}
@@ -87,24 +82,11 @@ func main() {
}
for _, spike := range suppress.Filter(detected) {
fired++
var cls []classification
for _, p := range spike.Patterns {
c := classification{}
if *needle != "" {
if hit, ok := cache[p.Signature]; ok {
c = hit
} else {
c = classify(*needle, needleText(p.Example))
cache[p.Signature] = c
}
}
cls = append(cls, c)
}
if *asJSON {
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
continue
}
printSpike(spike, cls)
printSpike(spike)
}
}
for c, n := range muted {
@@ -113,28 +95,7 @@ func main() {
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
}
type classification struct {
Action string `json:"action"`
Confidence float64 `json:"confidence"`
Cached bool `json:"cached"`
Err string `json:"error,omitempty"`
}
func classify(base, text string) classification {
body, _ := json.Marshal(map[string]string{"text": text})
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
if err != nil {
return classification{Action: "?", Err: err.Error()}
}
defer resp.Body.Close()
var out classification
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return classification{Action: "?", Err: err.Error()}
}
return out
}
func needleText(e features.Event) string {
func eventText(e features.Event) string {
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
if e.SpanName != "" {
parts = append(parts, "Title: "+e.SpanName)
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
return strings.Join(parts, " | ")
}
func printSpike(s features.Spike, cls []classification) {
func printSpike(s features.Spike) {
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
for i, p := range s.Patterns {
for _, p := range s.Patterns {
tag := "rising"
if p.Novel {
tag = "NEW"
}
what := string(alerts.TriageOf(p.Example).Category)
if cls[i].Action != "" {
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
}
clients := ""
if p.Clients > 0 {
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
clients += " [" + p.Example.UAClass + "]"
}
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
}
}
cmd/needle-sidecar/eval/golden.jsonl → cmd/triage-eval/data/golden.jsonl
View File
File renamed without changes.
cmd/needle-sidecar/eval/holdout.jsonl → cmd/triage-eval/data/holdout.jsonl
View File
File renamed without changes.