Remove Needle; stop paging on nginx 4xx
Needle scored ~chance on the golden set and never gated alerts, so drop the sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled eval sets move to cmd/triage-eval/data. nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped in FilterActionable instead of only being discounted, so it cannot ride along with a real 5xx. nginx 5xx still alerts. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e046bd86d1
commit
51a8412ae1
22 files changed
+29
-1162
No files matched your search
@@ -1,16 +1,13 @@
|
||||
// spike-alerts replays the deterministic spike rule over a bounded window of
|
||||
// normalized_events and, optionally, asks the Needle sidecar what kind of
|
||||
// failure each top pattern is. It only reads ClickHouse.
|
||||
// normalized_events. It only reads ClickHouse.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -36,7 +33,6 @@ func main() {
|
||||
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
|
||||
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
|
||||
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
|
||||
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
|
||||
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
|
||||
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
|
||||
flag.Parse()
|
||||
@@ -75,7 +71,6 @@ func main() {
|
||||
log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
|
||||
log.Printf("loaded %d error events", len(events))
|
||||
|
||||
cache := map[string]classification{}
|
||||
first := start.Add(time.Duration(*baselineWindows) * *window)
|
||||
suppress := features.NewSuppressor(*cooldown)
|
||||
muted := map[triage.Category]int{}
|
||||
@@ -87,24 +82,11 @@ func main() {
|
||||
}
|
||||
for _, spike := range suppress.Filter(detected) {
|
||||
fired++
|
||||
var cls []classification
|
||||
for _, p := range spike.Patterns {
|
||||
c := classification{}
|
||||
if *needle != "" {
|
||||
if hit, ok := cache[p.Signature]; ok {
|
||||
c = hit
|
||||
} else {
|
||||
c = classify(*needle, needleText(p.Example))
|
||||
cache[p.Signature] = c
|
||||
}
|
||||
}
|
||||
cls = append(cls, c)
|
||||
}
|
||||
if *asJSON {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
|
||||
continue
|
||||
}
|
||||
printSpike(spike, cls)
|
||||
printSpike(spike)
|
||||
}
|
||||
}
|
||||
for c, n := range muted {
|
||||
@@ -113,28 +95,7 @@ func main() {
|
||||
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
type classification struct {
|
||||
Action string `json:"action"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Cached bool `json:"cached"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func classify(base, text string) classification {
|
||||
body, _ := json.Marshal(map[string]string{"text": text})
|
||||
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return classification{Action: "?", Err: err.Error()}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var out classification
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return classification{Action: "?", Err: err.Error()}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func needleText(e features.Event) string {
|
||||
func eventText(e features.Event) string {
|
||||
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
|
||||
if e.SpanName != "" {
|
||||
parts = append(parts, "Title: "+e.SpanName)
|
||||
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
|
||||
return strings.Join(parts, " | ")
|
||||
}
|
||||
|
||||
func printSpike(s features.Spike, cls []classification) {
|
||||
func printSpike(s features.Spike) {
|
||||
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
|
||||
for i, p := range s.Patterns {
|
||||
for _, p := range s.Patterns {
|
||||
tag := "rising"
|
||||
if p.Novel {
|
||||
tag = "NEW"
|
||||
}
|
||||
what := string(alerts.TriageOf(p.Example).Category)
|
||||
if cls[i].Action != "" {
|
||||
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
|
||||
}
|
||||
clients := ""
|
||||
if p.Clients > 0 {
|
||||
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
|
||||
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
|
||||
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
||||
clients += " [" + p.Example.UAClass + "]"
|
||||
}
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user