mats/go-otel
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Needle; stop paging on nginx 4xx

Browse Source 
Needle scored ~chance on the golden set and never gated alerts, so drop the
sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled
eval sets move to cmd/triage-eval/data.

nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped
in FilterActionable instead of only being discounted, so it cannot ride along
with a real 5xx. nginx 5xx still alerts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed 2026-11-01 11:40:53 +01:00
1 parent e046bd86d1
commit 51a8412ae1
22 files changed
+29 -1162
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/run_needle_test.sh
-21
View File
@@ -1,21 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
# Use pre-configured virtual environment or fall back to local .venv
VENV_DIR="/home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv"
if [ ! -d "${VENV_DIR}" ]; then
VENV_DIR="${ROOT_DIR}/.venv"
fi
if [ ! -f "${VENV_DIR}/bin/python" ]; then
echo "Setting up virtual environment at ${VENV_DIR}..."
python3 -m venv "${VENV_DIR}"
"${VENV_DIR}/bin/pip" install --upgrade pip
"${VENV_DIR}/bin/pip" install cactus-needle pydantic
fi
echo "Running Needle 3 OTel Verification Suite..."
"${VENV_DIR}/bin/python" "${SCRIPT_DIR}/test_needle_classifier.py"
scripts/test_needle_classifier.py
-168
View File
@@ -1,168 +0,0 @@
#!/usr/bin/env python3
"""
Test and Verification Suite for Cactus-Compute/needle3 with OpenTelemetry data.
Tests:
1. Action/Tool Dispatch Accuracy (Ground Truth Verification)
2. Field Extraction & Grounding
3. Confidence Calibrations (Act vs. Escalate)
4. Semantic Vector Embeddings (Issue Clustering)
5. Execution Performance & Latency (Tokens/sec)
"""
import sys
import time
import json
import math
import subprocess
try:
import needle
except ImportError:
print("Error: cactus-needle not found. Run with the virtualenv python:")
print(" /home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv/bin/python " + __file__)
sys.exit(1)
# --- Define Triage Tools ---
@needle.tool
def report_client_javascript_crash(service: str, message: str, url: str):
"""Report client-side JavaScript crash or error logged by browser RUM."""
return {"action": "client_js_crash", "service": service, "message": message, "url": url}
@needle.tool
def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
"""Alert on malicious vulnerability scans or exploit crawlers."""
return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
@needle.tool
def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
"""Record RUM performance metrics like TTFB or FCP."""
return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
@needle.tool
def log_missing_resource_404(path: str, host: str):
"""Log benign 404 Not Found requests for missing pages or git commits."""
return {"action": "log_404", "path": path, "host": host}
@needle.tool
def escalate_backend_outage(service: str, error_message: str):
"""Escalate severe backend database timeout or service crash to on-call."""
return {"action": "escalate_outage", "service": service, "error": error_message}
def run_verification():
print("=" * 70)
print(" NEEDLE 3 OPENTELEMETRY CLASSIFIER - VERIFICATION SUITE")
print("=" * 70)
# 1. Initialize Agent
print("\n[Step 1] Initializing Needle 3 Engine...")
t0 = time.perf_counter()
agent = needle.Needle(tools=[
report_client_javascript_crash,
alert_vulnerability_probe,
record_frontend_web_vital,
log_missing_resource_404,
escalate_backend_outage
])
init_ms = (time.perf_counter() - t0) * 1000
print(f"✓ Needle 3 Engine initialized in {init_ms:.1f}ms")
# 2. Ground-Truth Test Cases (from actual ClickHouse telemetry)
test_cases = [
{
"name": "Frontend Browser RUM Crash",
"input": "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
"expected_tool": "report_client_javascript_crash",
"expected_fields": ["service", "url"]
},
{
"name": "Malicious CVE Exploit Probe",
"input": "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
"expected_tool": "alert_vulnerability_probe",
"expected_fields": ["target", "status_code"]
},
{
"name": "Web Vital RUM Metric",
"input": "Store RUM metric on service storefront-web: record TTFB of 84 ms",
"expected_tool": "record_frontend_web_vital",
"expected_fields": ["service", "metric", "ms"]
},
{
"name": "Nginx Benign 404 Route",
"input": "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
"expected_tool": "log_missing_resource_404",
"expected_fields": ["path", "host"]
},
{
"name": "Critical Database Outage",
"input": "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
"expected_tool": "escalate_backend_outage",
"expected_fields": ["service"]
}
]
print("\n[Step 2] Testing Classification & Tool Dispatch Accuracy...")
passed = 0
total = len(test_cases)
latencies = []
for i, tc in enumerate(test_cases, 1):
agent.reset()
t_start = time.perf_counter()
res = agent.run(tc["input"], strict=False)
dur_ms = (time.perf_counter() - t_start) * 1000
latencies.append(dur_ms)
results = res.get("results", [])
matched_action = results[0].get("action") if results and isinstance(results[0], dict) else None
# Tool name to action map
expected_action = tc["expected_tool"].replace("report_", "").replace("alert_", "").replace("record_", "").replace("log_", "").replace("escalate_", "")
# Check match
is_match = False
if results and any(tc["expected_tool"] in str(c) or matched_action for c in results):
is_match = True
status_str = "PASS" if is_match else "FAIL"
if is_match:
passed += 1
print(f"\n Test {i}/{total}: {tc['name']} -> [{status_str}] ({dur_ms:.1f}ms)")
print(f" Confidence: {res.get('confidence', 0.0):.3f}")
print(f" Reasoning : {res.get('reasoning')}")
print(f" Dispatched: {results}")
print("\n" + "-" * 70)
print(f"Accuracy Score: {passed}/{total} ({passed/total*100:.0f}%) | Avg Latency: {sum(latencies)/len(latencies):.1f}ms")
print("-" * 70)
# 3. Embedding & Semantic Search Verification
print("\n[Step 3] Testing Semantic Vector Embeddings for Near-Match Clustering...")
emb_e1 = agent.embed("database connection timeout after 3000ms")
emb_e2 = agent.embed("postgres pool exhausted: connection timed out")
emb_e3 = agent.embed("invalid JWT signature token expired")
def cosine_similarity(v1, v2):
dot = sum(a * b for a, b in zip(v1, v2))
m1 = math.sqrt(sum(a * a for a in v1))
m2 = math.sqrt(sum(b * b for b in v2))
return dot / (m1 * m2)
sim_similar = cosine_similarity(emb_e1, emb_e2)
sim_different = cosine_similarity(emb_e1, emb_e3)
print(f" Vector Dimensions: {len(emb_e1)}")
print(f" Similarity between 2 DB timeouts : {sim_similar:.4f}")
print(f" Similarity between DB and Auth : {sim_different:.4f}")
if sim_similar > sim_different:
print("✓ Semantic clustering validated: related issues score higher similarity.")
else:
print("✗ Warning: semantic separation failed.")
print("\n" + "=" * 70)
print(" VERIFICATION COMPLETE")
print("=" * 70)
if __name__ == "__main__":
run_verification()