Remove Needle; stop paging on nginx 4xx

Needle scored ~chance on the golden set and never gated alerts, so drop the
sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled
eval sets move to cmd/triage-eval/data.

nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped
in FilterActionable instead of only being discounted, so it cannot ride along
with a real 5xx. nginx 5xx still alerts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed 2026-11-01 11:40:53 +01:00
1 parent e046bd86d1
commit 51a8412ae1
22 files changed
+29 -1162

No files matched your search

+7 -49
View File
@@ -1,16 +1,13 @@
// spike-alerts replays the deterministic spike rule over a bounded window of
// normalized_events and, optionally, asks the Needle sidecar what kind of
// failure each top pattern is. It only reads ClickHouse.
// normalized_events. It only reads ClickHouse.
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"strings"
"time"
@@ -36,7 +33,6 @@ func main() {
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
flag.Parse()
@@ -75,7 +71,6 @@ func main() {
log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
log.Printf("loaded %d error events", len(events))
cache := map[string]classification{}
first := start.Add(time.Duration(*baselineWindows) * *window)
suppress := features.NewSuppressor(*cooldown)
muted := map[triage.Category]int{}
@@ -87,24 +82,11 @@ func main() {
}
for _, spike := range suppress.Filter(detected) {
fired++
var cls []classification
for _, p := range spike.Patterns {
c := classification{}
if *needle != "" {
if hit, ok := cache[p.Signature]; ok {
c = hit
} else {
c = classify(*needle, needleText(p.Example))
cache[p.Signature] = c
}
}
cls = append(cls, c)
}
if *asJSON {
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
continue
}
printSpike(spike, cls)
printSpike(spike)
}
}
for c, n := range muted {
@@ -113,28 +95,7 @@ func main() {
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
}
type classification struct {
Action string `json:"action"`
Confidence float64 `json:"confidence"`
Cached bool `json:"cached"`
Err string `json:"error,omitempty"`
}
func classify(base, text string) classification {
body, _ := json.Marshal(map[string]string{"text": text})
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
if err != nil {
return classification{Action: "?", Err: err.Error()}
}
defer resp.Body.Close()
var out classification
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return classification{Action: "?", Err: err.Error()}
}
return out
}
func needleText(e features.Event) string {
func eventText(e features.Event) string {
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
if e.SpanName != "" {
parts = append(parts, "Title: "+e.SpanName)
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
return strings.Join(parts, " | ")
}
func printSpike(s features.Spike, cls []classification) {
func printSpike(s features.Spike) {
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
for i, p := range s.Patterns {
for _, p := range s.Patterns {
tag := "rising"
if p.Novel {
tag = "NEW"
}
what := string(alerts.TriageOf(p.Example).Category)
if cls[i].Action != "" {
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
}
clients := ""
if p.Clients > 0 {
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
clients += " [" + p.Example.UAClass + "]"
}
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
}
}