diff --git a/build.sh b/build.sh index 38c3514..773d2ac 100755 --- a/build.sh +++ b/build.sh @@ -5,10 +5,6 @@ RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}" PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}" UPLOAD_SOURCEMAPS="${UPLOAD_SOURCEMAPS:-true}" -echo "Building Needle 3 sidecar image..." -docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar -docker push matst80/needle-sidecar:latest - echo "Building otel-debug receiver image..." docker build \ --build-arg "APP_RELEASE=${RELEASE}" \ diff --git a/cmd/needle-sidecar/Dockerfile b/cmd/needle-sidecar/Dockerfile deleted file mode 100644 index 8c7f6e8..0000000 --- a/cmd/needle-sidecar/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.12-slim - -# Create non-root user with UID 1000 matching Kubernetes pod securityContext -RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser - -WORKDIR /app - -ENV HOME=/app -ENV NEEDLE_PORT=5005 -ENV NEEDLE_HOST=0.0.0.0 -ENV NEEDLE_TELEMETRY=0 - -# Install dependencies -RUN pip install --no-cache-dir cactus-needle - -# Pre-cache model and shared library into /app/.cache as user 1000 -RUN python -c "import needle; needle.Needle()" - -COPY server.py /app/server.py - -# Ensure UID 1000 owns /app and all caches -RUN chown -R appuser:appuser /app - -USER appuser - -EXPOSE 5005 - -CMD ["python", "/app/server.py"] diff --git a/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc b/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc deleted file mode 100644 index 8b783da..0000000 Binary files a/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc and /dev/null differ diff --git a/cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc b/cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc deleted file mode 100644 index 60a6ecd..0000000 Binary files a/cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc and /dev/null differ diff --git a/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc b/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc deleted file mode 100644 index dea1707..0000000 Binary files a/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc and /dev/null differ diff --git a/cmd/needle-sidecar/eval/rules.py b/cmd/needle-sidecar/eval/rules.py deleted file mode 100644 index 9ea4a1c..0000000 --- a/cmd/needle-sidecar/eval/rules.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Deterministic baseline for the cause categories. Needle has to beat this.""" -import re - -PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I) -DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I) -CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I) -BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I) -LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I) - - -def fields(text): - d = {} - for part in text.split(" | "): - k, _, v = part.partition(": ") - d[k.lower()] = v - return d - - -def classify(text): - f = fields(text) - status = int(f.get("status") or 0) - route = f.get("route", "") - blob = " ".join([f.get("title", ""), f.get("message", ""), route]) - if PROBE.search(route): - return "scanner_probe" - if BUG.search(blob): - return "service_defect" - if CAP.search(blob): - return "capacity_saturation" - if DEP.search(blob): - return "dependency_unreachable" - if status == 401: - return "client_credential_issue" - if status in (502, 503, 504): - return "dependency_unreachable" - if status >= 500: - return "service_defect" - if status == 404: - return "stale_reference" - if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")): - return "expected_lifecycle" - return "needs_review" diff --git a/cmd/needle-sidecar/eval/run_eval.py b/cmd/needle-sidecar/eval/run_eval.py deleted file mode 100755 index b967013..0000000 --- a/cmd/needle-sidecar/eval/run_eval.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate the Needle sidecar against a reviewed golden set. - -Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json] - -Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name -including "needs_review" for the fallback. -""" -import argparse, json, os, sys, time, urllib.request -from collections import Counter, defaultdict - -HERE = os.path.dirname(os.path.abspath(__file__)) - - -def classify(url, text): - req = urllib.request.Request( - url.rstrip("/") + "/classify", - data=json.dumps({"text": text}).encode(), - headers={"Content-Type": "application/json"}, - ) - with urllib.request.urlopen(req, timeout=60) as r: - return json.load(r) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005")) - ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl")) - ap.add_argument("--json", help="write per-row results here") - ap.add_argument("--min-conf", type=float, default=0.0, - help="predictions below this confidence count as needs_review") - a = ap.parse_args() - - rows = [json.loads(l) for l in open(a.golden) if l.strip()] - results, confusion = [], Counter() - lat = [] - for row in rows: - try: - out = classify(a.url, row["text"]) - except Exception as e: - print(f"sidecar error on {row['id']}: {e}", file=sys.stderr) - sys.exit(2) - pred = out.get("action", "needs_review") - conf = out.get("confidence", 0.0) - if conf < a.min_conf: - pred = "needs_review" - lat.append(out.get("latency_ms", 0.0)) - confusion[(row["expected"], pred)] += 1 - results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]}) - - labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results}) - print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}") - for l in labels: - tp = confusion[(l, l)] - pp = sum(v for (e, p), v in confusion.items() if p == l) - sup = sum(v for (e, p), v in confusion.items() if e == l) - prec = tp / pp if pp else 0.0 - rec = tp / sup if sup else 0.0 - print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}") - - acc = sum(r["ok"] for r in results) / len(results) - print(f"\naccuracy {acc:.2%} over {len(results)} rows; " - f"median latency {sorted(lat)[len(lat)//2]:.0f}ms") - - for src in sorted({r.get("src", "?") for r in results}): - sub = [r for r in results if r.get("src", "?") == src] - print(f" {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}") - - wrong = [r for r in results if not r["ok"]] - if wrong: - print("\nmisses:") - for r in wrong: - print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}") - if a.json: - json.dump(results, open(a.json, "w"), indent=1) - - -if __name__ == "__main__": - main() diff --git a/cmd/needle-sidecar/server.py b/cmd/needle-sidecar/server.py deleted file mode 100755 index 7da5bff..0000000 --- a/cmd/needle-sidecar/server.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -""" -Needle 3 AI Telemetry Triage Sidecar Service -Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights. -""" - -import os -import sys -import json -import time -import re -import hashlib -from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler -import needle - -PORT = int(os.environ.get("NEEDLE_PORT", 5005)) -HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0") - -# --- Cause-level triage tools --- -# These deliberately do not restate what the event already carries (status -# code, is_error, severity). Each one is a judgement about *why* it happened -# or *who should care*, which is what an alert reader needs. -@needle.tool -def report_scanner_probe(service: str, path: str, status_code: int): - """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502.""" - return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code} - -@needle.tool -def report_dependency_unreachable(service: str, dependency: str, error_message: str): - """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable.""" - return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message} - -@needle.tool -def report_client_credential_issue(service: str, route: str, reason: str): - """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service.""" - return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason} - -@needle.tool -def report_stale_reference(service: str, path: str): - """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request.""" - return {"action": "stale_reference", "service": service, "path": path} - -@needle.tool -def report_service_defect(service: str, error_message: str): - """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route.""" - return {"action": "service_defect", "service": service, "error": error_message} - -@needle.tool -def report_capacity_saturation(service: str, resource: str, error_message: str): - """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full.""" - return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message} - -@needle.tool -def report_expected_lifecycle(service: str, what: str): - """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals.""" - return {"action": "expected_lifecycle", "service": service, "what": what} - -@needle.tool -def report_needs_review(service: str, summary: str): - """Fallback when none of the other categories clearly applies.""" - return {"action": "needs_review", "service": service, "summary": summary} - -# --- Redaction: nothing PII-bearing should reach the model or the cache --- -_REDACTIONS = [ - (re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), ""), - (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), ""), - (re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), ""), - (re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), ""), - (re.compile(r"\b[0-9a-fA-F]{16,}\b"), ""), - (re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1"), -] - -def redact(text: str) -> str: - for rx, sub in _REDACTIONS: - text = rx.sub(sub, text) - return text[:1000] - -# Cache keyed on the redacted text: repeated signatures cost nothing. -CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000)) -_cache = {} -_stats = {"hits": 0, "misses": 0} - -# Initialize Needle Agent once -print(f"Loading Needle 3 model (Cactus-Compute/needle3)...") -t0 = time.perf_counter() -agent = needle.Needle(tools=[ - report_scanner_probe, - report_dependency_unreachable, - report_client_credential_issue, - report_stale_reference, - report_service_defect, - report_capacity_saturation, - report_expected_lifecycle, - report_needs_review, -]) -print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s") - - -class NeedleHandler(BaseHTTPRequestHandler): - def _send_cors(self): - self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token") - - def do_OPTIONS(self): - self.send_response(204) - self._send_cors() - self.end_headers() - - def do_GET(self): - if self.path == "/health" or self.path == "/api/needle/status": - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - resp = { - "status": "ok", - "available": True, - "model": "Cactus-Compute/needle3", - "cache": {**_stats, "size": len(_cache)}, - "tools": [ - "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference", - "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review" - ] - } - self.wfile.write(json.dumps(resp).encode("utf-8")) - else: - self.send_response(404) - self._send_cors() - self.end_headers() - - def do_POST(self): - if self.path in ("/classify", "/api/needle/classify"): - content_len = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(content_len) - try: - data = json.loads(body) if body else {} - except Exception as e: - self.send_response(400) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8")) - return - - text = data.get("text", "").strip() - # If text is not provided, build from structured fields: - if not text: - parts = [] - if data.get("service"): - parts.append(f"Service: {data['service']}") - if data.get("kind"): - parts.append(f"Kind: {data['kind']}") - if data.get("title"): - parts.append(f"Title: {data['title']}") - if data.get("message"): - parts.append(f"Message: {data['message']}") - if data.get("url"): - parts.append(f"URL: {data['url']}") - if data.get("status_code"): - parts.append(f"Status: {data['status_code']}") - text = " | ".join(parts) - - if not text: - self.send_response(400) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8")) - return - - text = redact(text) - key = hashlib.sha256(text.encode()).hexdigest() - cached = _cache.get(key) - if cached is not None: - _stats["hits"] += 1 - out = {**cached, "cached": True} - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps(out).encode("utf-8")) - return - _stats["misses"] += 1 - - t_start = time.perf_counter() - agent.reset() - res = agent.run(text, strict=False) - latency_ms = (time.perf_counter() - t_start) * 1000 - - results = res.get("results", []) - first_result = results[0] if results and isinstance(results[0], dict) else {} - action = first_result.get("action", "needs_review") - - calls = res.get("function_calls", []) - tool_name = calls[0].get("name") if calls else "triage_tool" - - out = { - "success": True, - "input_text": text, - "action": action, - "tool": tool_name, - "arguments": first_result, - "confidence": float(res.get("confidence", 0.0)), - "reasoning": res.get("reasoning", ""), - "latency_ms": round(latency_ms, 2), - "model": "Cactus-Compute/needle3" - } - - if len(_cache) >= CACHE_MAX: - _cache.pop(next(iter(_cache))) - _cache[key] = out - - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps(out).encode("utf-8")) - else: - self.send_response(404) - self._send_cors() - self.end_headers() - - def log_message(self, format, *args): - # Concise logging - print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}") - - -def run(): - server = ThreadingHTTPServer((HOST, PORT), NeedleHandler) - print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}") - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nShutting down needle sidecar...") - server.server_close() - - -if __name__ == "__main__": - run() diff --git a/cmd/server/main.go b/cmd/server/main.go index 5956ec9..f987020 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -490,8 +490,6 @@ func main() { uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap) uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps))) uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks))) - uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus) - uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify) // Protected API & WebSocket routes uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory))) diff --git a/cmd/spike-alerts/main.go b/cmd/spike-alerts/main.go index 742d483..3b8de36 100644 --- a/cmd/spike-alerts/main.go +++ b/cmd/spike-alerts/main.go @@ -1,16 +1,13 @@ // spike-alerts replays the deterministic spike rule over a bounded window of -// normalized_events and, optionally, asks the Needle sidecar what kind of -// failure each top pattern is. It only reads ClickHouse. +// normalized_events. It only reads ClickHouse. package main import ( - "bytes" "context" "encoding/json" "flag" "fmt" "log" - "net/http" "os" "strings" "time" @@ -36,7 +33,6 @@ func main() { minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert") multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline") cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles") - needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it") showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)") asJSON := flag.Bool("json", false, "emit JSON lines instead of text") flag.Parse() @@ -75,7 +71,6 @@ func main() { log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events)) log.Printf("loaded %d error events", len(events)) - cache := map[string]classification{} first := start.Add(time.Duration(*baselineWindows) * *window) suppress := features.NewSuppressor(*cooldown) muted := map[triage.Category]int{} @@ -87,24 +82,11 @@ func main() { } for _, spike := range suppress.Filter(detected) { fired++ - var cls []classification - for _, p := range spike.Patterns { - c := classification{} - if *needle != "" { - if hit, ok := cache[p.Signature]; ok { - c = hit - } else { - c = classify(*needle, needleText(p.Example)) - cache[p.Signature] = c - } - } - cls = append(cls, c) - } if *asJSON { - _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls}) + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike}) continue } - printSpike(spike, cls) + printSpike(spike) } } for c, n := range muted { @@ -113,28 +95,7 @@ func main() { log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339)) } -type classification struct { - Action string `json:"action"` - Confidence float64 `json:"confidence"` - Cached bool `json:"cached"` - Err string `json:"error,omitempty"` -} - -func classify(base, text string) classification { - body, _ := json.Marshal(map[string]string{"text": text}) - resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body)) - if err != nil { - return classification{Action: "?", Err: err.Error()} - } - defer resp.Body.Close() - var out classification - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return classification{Action: "?", Err: err.Error()} - } - return out -} - -func needleText(e features.Event) string { +func eventText(e features.Event) string { parts := []string{"Service: " + e.Service, "Kind: " + e.Kind} if e.SpanName != "" { parts = append(parts, "Title: "+e.SpanName) @@ -151,17 +112,14 @@ func needleText(e features.Event) string { return strings.Join(parts, " | ") } -func printSpike(s features.Spike, cls []classification) { +func printSpike(s features.Spike) { fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess) - for i, p := range s.Patterns { + for _, p := range s.Patterns { tag := "rising" if p.Novel { tag = "NEW" } what := string(alerts.TriageOf(p.Example).Category) - if cls[i].Action != "" { - what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence) - } clients := "" if p.Clients > 0 { clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1]) @@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) { if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) { clients += " [" + p.Example.UAClass + "]" } - fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients) + fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients) } } diff --git a/cmd/needle-sidecar/eval/golden.jsonl b/cmd/triage-eval/data/golden.jsonl similarity index 100% rename from cmd/needle-sidecar/eval/golden.jsonl rename to cmd/triage-eval/data/golden.jsonl diff --git a/cmd/needle-sidecar/eval/holdout.jsonl b/cmd/triage-eval/data/holdout.jsonl similarity index 100% rename from cmd/needle-sidecar/eval/holdout.jsonl rename to cmd/triage-eval/data/holdout.jsonl diff --git a/docs/ML.md b/docs/ML.md index d3a5dda..6bad9eb 100644 --- a/docs/ML.md +++ b/docs/ML.md @@ -1,7 +1,7 @@ # Error detection model > Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they -> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins. +> differ (ClickHouse is now a LoadBalancer; Needle is removed), STATUS.md wins. ## Current handoff (2026-09-18) diff --git a/docs/STATUS.md b/docs/STATUS.md index 320a057..cf7445c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -10,7 +10,6 @@ unless stated. Greenfield, single user: radical changes are fine. patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded. - Credentials are in `.env` (gitignored, mode 600). Load with `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally. -- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly. ## What exists now @@ -21,7 +20,7 @@ unless stated. Greenfield, single user: radical changes are fine. | 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. | | Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). | | Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. | -| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. | +| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories. | | Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. | | Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. | | Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. | @@ -35,14 +34,14 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` ## Findings worth remembering -- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes). - Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`), - so it picks by position, not content. Confidence is uncorrelated with - correctness. Decision: **Needle stays dormant.** Intended later use: a gate that - decides whether a larger model should act on an issue (affects anyone? fixable by - a smart model?). Do not use it for categorization. +- **Needle was removed (2026-11-01).** It scored 10% on the 40-row golden set (about + chance for 8 classes), picked by tool position rather than content, and its + confidence was uncorrelated with correctness. Sidecar, proxy handlers, UI and k8s + container are gone; the labelled sets live in `cmd/triage-eval/data/`. +- **Edge 4xx never pages.** nginx 4xx (registry 401 challenges, `/oauth2/auth`, + scanner 405s) is dropped in `FilterActionable`; nginx 5xx still alerts. - **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing - the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl` + the rows). A fresh 60-row holdout is in `cmd/triage-eval/data/holdout.jsonl` (from 2026-09-18, unseen); it still needs your `expected` labels. Known probe-like paths that still land in `stale_reference`: `/ssl/server.key`, `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`, @@ -78,7 +77,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` 1. Deploy the collector so normalized rows stay current; verify with `SELECT max(timestamp) FROM otel.normalized_events`. 2. Label `holdout.jsonl` (fill `expected`), run - `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix + `go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl`, fix the rule families that miss. Then re-export a fresh holdout so scoring stays honest. 3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the @@ -164,7 +163,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` ```bash set -a; . ./.env; set +a -# replay the spike rule over a range (Needle off) +# replay the spike rule over a range go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12 # add -show-muted to see probes / stale / lifecycle patterns too @@ -172,12 +171,7 @@ go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00 go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z # score the rules against labelled rows -go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl - -# Needle eval (needs the sidecar running; venv was in scratch and is gone) -pip install cactus-needle # python 3.12 recommended -NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py -python cmd/needle-sidecar/eval/run_eval.py +go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl ``` Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows @@ -187,14 +181,11 @@ through a `kubectl port-forward` hung and dropped. Everything is uncommitted. Files changed by this work: `internal/{storage, normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts, -triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`, +triage-eval,bad-actors,normalized-backfill,server}`, `k8s/{clickhouse,deployment}.yaml`, `.gitignore`. **Not from this work; check before committing:** `AGENTS.md`, -`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` -(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files -under `cmd/needle-sidecar` show as modified; they should be untracked and -gitignored. +`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` (including a deleted `GaugeMeter.tsx`). ## IP details (MaxMind) diff --git a/index.html b/index.html index 2ec9bdb..00e424e 100644 --- a/index.html +++ b/index.html @@ -141,45 +141,9 @@ Clear Logs - -
-

Needle 3 AI Telemetry Triage

-

Test on-device telemetry classification with Cactus-Compute/needle3:

-
- - -
-
- Click "Classify with Needle 3" to test the sidecar... -
-