From f30adf05af6d1aaa97b9977fe67ab6cc50f15643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats=20T=C3=B6rnberg?= Date: Sat, 19 Sep 2026 23:25:59 +0200 Subject: [PATCH] all the changes --- .gitignore | 3 + AGENTS.md | 5 +- cmd/bad-actors/main.go | 184 ++ .../__pycache__/server.cpython-314.pyc | Bin 15054 -> 15553 bytes .../eval/__pycache__/rules.cpython-314.pyc | Bin 0 -> 3012 bytes .../eval/__pycache__/run_eval.cpython-314.pyc | Bin 7046 -> 6936 bytes cmd/needle-sidecar/eval/golden.jsonl | 80 +- cmd/needle-sidecar/eval/holdout.jsonl | 60 + cmd/needle-sidecar/eval/rules.py | 42 + cmd/needle-sidecar/eval/run_eval.py | 11 +- cmd/needle-sidecar/server.py | 81 +- cmd/normalized-backfill/main.go | 73 +- cmd/server/main.go | 10 + cmd/spike-alerts/main.go | 49 +- cmd/triage-eval/main.go | 98 + docs/ML.md | 3 + docs/STATUS.md | 132 ++ internal/blocker/blocker.go | 383 ++++ internal/blocker/blocker_test.go | 153 ++ internal/collector/collector_test.go | 41 + internal/collector/issue_extract.go | 18 +- internal/honeypot/honeypot.go | 73 +- internal/honeypot/honeypot_test.go | 30 +- internal/storage/clickhouse.go | 4 + internal/storage/normalized.go | 107 + internal/triage/triage.go | 107 + internal/triage/triage_test.go | 43 + k8s/clickhouse.yaml | 4 +- k8s/deployment.yaml | 39 + otel-ui/src/App.tsx | 8 +- otel-ui/src/components/AnomalyRadarBar.tsx | 2 +- otel-ui/src/components/GaugeMeter.tsx | 372 ---- otel-ui/src/components/MetricsVisualizer.tsx | 1851 ++++++++--------- otel-ui/src/components/RolloutsVisualizer.tsx | 497 ++++- otel-ui/src/hooks/useRollouts.ts | 23 +- 35 files changed, 2979 insertions(+), 1607 deletions(-) create mode 100644 cmd/bad-actors/main.go create mode 100644 cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc create mode 100644 cmd/needle-sidecar/eval/holdout.jsonl create mode 100644 cmd/needle-sidecar/eval/rules.py create mode 100644 cmd/triage-eval/main.go create mode 100644 docs/STATUS.md create mode 100644 internal/blocker/blocker.go create mode 100644 internal/blocker/blocker_test.go create mode 100644 internal/storage/normalized.go create mode 100644 internal/triage/triage.go create mode 100644 internal/triage/triage_test.go delete mode 100644 otel-ui/src/components/GaugeMeter.tsx diff --git a/.gitignore b/.gitignore index bd0b69c..f13d797 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ otel-agent.log /otel-ui/node_modules/ /otel-ui/dist/ /otel-ui/dist-ssr/ + +# Local credentials and scratch +.env diff --git a/AGENTS.md b/AGENTS.md index a1f6a09..bcbdb72 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,11 +149,12 @@ Lessons from getting here: `index.html`. Rebuild (`npm --prefix otel-ui run build`) before concluding a screen isn't wired. - Still to polish (functional, but not yet re-laid-out to the screen - pattern): `TraceVisualization`, `MetricsVisualizer`, `StructuredLogTerminal`, + pattern): `TraceVisualization`, `StructuredLogTerminal`, `SearchAndWatcher`, `TrendsVisualizer`, `SetupPanel`. They use `ds-*` colors via a mechanical token swap but keep their old layout/type scale, use `onClick` on non-buttons in places, and `CommandAndQueryBar` / - `WaveformHistogram` still sit above every tab except Issues and RUM sessions (`isTriageOverview` in `App.tsx`). Restructure + `WaveformHistogram` still sit above every tab except Issues, logs, RUM + sessions, rollouts and metrics (`isTriageOverview` in `App.tsx`). Restructure them onto `FilterBar` + the card/aside pattern when touching them. ### Finding old-pattern usages diff --git a/cmd/bad-actors/main.go b/cmd/bad-actors/main.go new file mode 100644 index 0000000..031a6e1 --- /dev/null +++ b/cmd/bad-actors/main.go @@ -0,0 +1,184 @@ +// bad-actors ranks client IPs by scanner behaviour in raw nginx/ingress spans +// and reports which would be safe to block. It only reads ClickHouse; nothing +// is blocked. Review its output before feeding any blocker. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + + "git.tornberg.me/go-otel/internal/honeypot" + "git.tornberg.me/go-otel/internal/triage" +) + +type otlp struct { + ResourceSpans []struct { + ScopeSpans []struct { + Spans []struct { + Attributes []struct { + Key string `json:"key"` + Value map[string]json.RawMessage `json:"value"` + } `json:"attributes"` + } `json:"spans"` + } `json:"scopeSpans"` + } `json:"resourceSpans"` +} + +type actor struct { + IP string + Total int + Probes int + Paths map[string]struct{} + Statuses map[string]int + UserAgent string +} + +func main() { + addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port") + db := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "database") + user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "user") + pass := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "password") + since := flag.String("since", "", "inclusive UTC RFC3339") + until := flag.String("until", "", "exclusive UTC RFC3339") + service := flag.String("service", "nginx", "ingress service name") + minProbes := flag.Int("min-probes", 10, "minimum probe requests to be blockable") + minPaths := flag.Int("min-paths", 3, "minimum distinct probe paths") + minShare := flag.Float64("min-share", 0.8, "minimum share of the IP's requests that are probes") + broadPaths := flag.Int("broad-paths", 50, "distinct probe paths that mark a scanner regardless of share") + broadProbes := flag.Int("broad-probes", 100, "probe requests required with -broad-paths") + top := flag.Int("top", 30, "rows to print") + flag.Parse() + start, err1 := time.Parse(time.RFC3339, *since) + end, err2 := time.Parse(time.RFC3339, *until) + if err1 != nil || err2 != nil || !start.Before(end) { + log.Fatal("-since/-until must be valid RFC3339 and ordered") + } + conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *db, Username: *user, Password: *pass}}) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT data FROM %s.entries WHERE type='traces' AND service=? AND timestamp >= toDateTime64('%s',9,'UTC') AND timestamp < toDateTime64('%s',9,'UTC')`, + *db, start.UTC().Format("2006-01-02 15:04:05.000000000"), end.UTC().Format("2006-01-02 15:04:05.000000000")), *service) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + actors := map[string]*actor{} + spans := 0 + for rows.Next() { + var data string + if err := rows.Scan(&data); err != nil { + log.Fatal(err) + } + var d otlp + if json.Unmarshal([]byte(data), &d) != nil { + continue + } + for _, rs := range d.ResourceSpans { + for _, ss := range rs.ScopeSpans { + for _, sp := range ss.Spans { + at := map[string]string{} + for _, a := range sp.Attributes { + for _, raw := range a.Value { + var v any + _ = json.Unmarshal(raw, &v) + at[a.Key] = fmt.Sprint(v) + } + } + ip := strings.TrimSpace(strings.SplitN(first(at, "http.request.header.x-real-ip", "http.request.header.x-forwarded-for", "client.address", "net.peer.ip"), ",", 2)[0]) + if ip == "" { + continue + } + spans++ + path := strings.SplitN(first(at, "http.target", "url.path", "http.route"), "?", 2)[0] + a := actors[ip] + if a == nil { + a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}} + actors[ip] = a + } + a.Total++ + if a.UserAgent == "" { + a.UserAgent = first(at, "http.user_agent") + } + if triage.IsProbePath(path) { + a.Probes++ + a.Paths[path] = struct{}{} + a.Statuses[first(at, "http.status_code")]++ + } + } + } + } + } + if err := rows.Err(); err != nil { + log.Fatal(err) + } + + var list []*actor + totalProbes, blockableProbes, blockable, skippedInternal, skippedInternalProbes := 0, 0, 0, 0, 0 + for _, a := range actors { + totalProbes += a.Probes + if a.Probes == 0 { + continue + } + if internal(a.IP) { + skippedInternal++ + skippedInternalProbes += a.Probes + a.UserAgent = "[INTERNAL] " + a.UserAgent + } else if (a.Probes >= *minProbes && len(a.Paths) >= *minPaths && float64(a.Probes)/float64(a.Total) >= *minShare) || + (a.Probes >= *broadProbes && len(a.Paths) >= *broadPaths) { + blockable++ + blockableProbes += a.Probes + a.UserAgent = "[BLOCK] " + a.UserAgent + } + list = append(list, a) + } + sort.Slice(list, func(i, j int) bool { return list[i].Probes > list[j].Probes }) + fmt.Printf("%d spans with an IP, %d distinct IPs, %d probe requests\n", spans, len(actors), totalProbes) + fmt.Printf("blockable (public; >=%d probes/>=%d paths/>=%.0f%% probes, or >=%d probes over >=%d paths): %d IPs covering %d probes (%.0f%%)\n", *minProbes, *minPaths, *minShare*100, *broadProbes, *broadPaths, blockable, blockableProbes, pct(blockableProbes, totalProbes)) + fmt.Printf("internal/loopback (never blocked): %d IPs, %d probes (%.0f%%) - real client is hidden behind a proxy hop\n\n", skippedInternal, skippedInternalProbes, pct(skippedInternalProbes, totalProbes)) + fmt.Printf("%-40s %6s %6s %6s %s\n", "ip", "probes", "total", "paths", "verdict / user-agent / statuses") + for i, a := range list { + if i >= *top { + break + } + fmt.Printf("%-40s %6d %6d %6d %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), a.UserAgent, a.Statuses) + } +} + +// internal reports addresses that must never be blocked; it is the same +// definition the tracker and blocker use. +func internal(s string) bool { return honeypot.Untrackable(s) } + +func first(m map[string]string, keys ...string) string { + for _, k := range keys { + if v := m[k]; v != "" && v != "" { + return v + } + } + return "" +} +func pct(a, b int) float64 { + if b == 0 { + return 0 + } + return 100 * float64(a) / float64(b) +} +func envOr(k, f string) string { + if v := os.Getenv(k); v != "" { + return v + } + return f +} diff --git a/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc b/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc index 898c190ca7d2da72d0100dc2035c128e9f9d130a..8b783da9e968364720cf53f8dba7af341b43abcf 100644 GIT binary patch delta 3639 zcmZ{nU2GKB6~}k>!(WTn#ModP3>Q-wjMpCoW6FY(nix`JFch1HP+&Wrxw|_sAIrTn zYppGsSiYL+Ii@MI}dFtHE)gNgh)}+OxCwYwrD>|2gOS&))B=nchreLk#|Y_Sri({?q+lW+5f!>zl+!sixh@vg($C zGdWn*|O;vF434eV^T&Kz2#Bv8g`LzYIDOiDip`rj!kC` z?o!(&xIIp|rfT-_nN_H zTNo`n%*}7=VXeSs2SVBtUVlo(u&$O3|b$MM{n11FTn-n#upQ#_#vRkh ziAhBz;HkU2n`PiT=#0Srp9K&H!sUCwyd0B+{q6nt>z9r$pY<{r*4-yErsQf{4ElRGx9U+=(qF_x` zyUB@4DynJdk}wFhY%ql8+@v7L0-&O^s#P{A0nq~^4{+M9p~*!8QAvA~uJ4a@Wa>rvJW=-`_tUU z!dwBYby@*;Ino1B3W|6&L9Q@q9-@OXh)v3Lsa&~t$tWH&!Ui4(vnJo|YkA>o^YN%C zHjav4r?mfG`~ByazPmi-r_cDwGm;Mui1=i2LWmXlOi4ad%3T@zQ_B02&s4}qjF4%s z7`h*#HjHzS2;>1cvQBf5Ky5<braCgS&f!4<6%P7=`2CvIR(O0 zyGW6ss+*1mXr)duQx=$GNy*NuCQ&itg1lh?A|oz>nTd9j*8!*IhU74aL%BsLWh!_a zW$PNs@GNVydtoAeHj@-*Gj+As{Pd)soLrClx;2T&JQKzJA)LDr_l|_>C<`*)9e|2) z?x($9&xXRWi^Jzy! zabvlS)dpc9{A>>0-g_@D)h>5DyzIYl-T%Jjr<1myv_YHSs|%7?Ohb^$u$XAxQ7>^# zDDUqRc5`lidBR~9NMV|m)QSP&3CT!xvD}0?3Ja)7tsE&))pVhf+d9Ol3Y8ZlTPsmW zt66fB(lR7gxC4|S(4hfp4ue__bX5pO)08mqXP8BGgNN9!pa&quK_b@h5pD|_EDTto zRbDZ$9~=em1Bn+LOsmi|L#Ii_fRN=8{btDKumsifrU9C*YNk-1YWvR92l3HIz4!A?EcJh&BLUhKIbRTA=PwH%RS3dSv3NC9Kc@596t@EW*X`YnSr_? z9yWKLfb&w7nULqgyG^v&EjFb(hh$YT=u9Y)qwwNQ(REZ@-PYE;X9NnqKn&YpKLpi zGt+RJ$%A(x#c*~`%(V|G#^Sr}AIFu`;-y_}xt%bGb)mr+WCzg=$Z7-(*Ye`ed2lj6TwQbVyp6MOr{fxgw(1!V5@ zkpbzwR9R&%4-z+Nm3c5ONPy&nTDYyc&=DjcEwWc|{*3rXpRW99akT%{6fE$^!_O(7 zh^8U0Rbsz{LrgTkLh~E(=FpjjF?>X$iO+^QI$KhEzQ|rtQhUCdjmJ}?3M5akY^xl# z_>CharAE$pTf8wm*6;>4qxn>PI{a1>l3=fa0liW76Y<*7ee+xJXD!_B4F*@WAL57) z(JY~P2~8^+yazJ8dN8c43=0CoR3^iYp|pcqsPtt6ZfnnCJ8G8w98E|LpWa6EI+!5t z4hA(7;I$l#zk|(k*K)t_;!iC4;i=>;weE!F=$=WRVIM(<0v~=|-1z1t*nYh3_Rd#! zei_IAj~*pr@y1p8xLWsEPi3R0qHO9ZZuAs4_l$4!jBoCl+UVIiwYkgL=yJrZW4i=D w_OESiE6Kep$*z@T$4at&CAoVg*(q-1hM$^Olu^ZRIrwEvK7BDN-ph6V4>aXFkpKVy delta 3303 zcmZ`*YiwLc72dmVd;LK1+K%nSp2n4l+j!T-k2r~4Nc~9b#CH7H29lV~-FwI0yS{tx zcJ5uj5OPALCh)5;1yvQ5f|MVw6sklnC{zK06ru3GKM+dq4^)XiAOUh80ttk~IdgaY zh#+#loqOiYnVE0S`DT8$_S<63vo+O~KKS>ypTC z-9VA)??`UIvKjwJDDhl9_*)$SSAGBV`K5X3BZ0Svx5e-McJKG$uxJ0o$8Sda=Z9x# z!DKd3bd4Hzm!+wcjLXaNtirScn3-&4iOC3M3{W>07pY1zYb3+W6^k-5lbT4lVOg;? z({O9ODCwM)76bXkNH^b()ehz$q9r1F9Sj#6#|5{F1#gFa)%8xpXE&SfoVoX|JtFhtpC3>ou7?xhvY*}YfnAZ(^LtH;B zr$i6h89{goV22)$9T8zfO)oIhu&JS@S7@db54V66?WN~+OlgAZi`GA;_t?LH-^ud zizGu0ZP6gT-MwVew8@xRG*rUqD@AJA76DF3Ub8ICSR@6gT9G!3m@;NQui52{i9lhc zAp|=Lv$9SEL4$wSVEMrc`5_26i2f%^fzymyW|1jWL^uX+?eO2itShC*>)OqR&G_c> z9ovZwI>A9m3@vWU8J)V-qOj8?QNiseaQgq134#Dsx-Du)l^*WNgwc|Us4_&AElObf zY|Rr5>ljg~tm>w+1Wn*WF`<5iV32}o>crOa)GY3BQBw|*7E2Qe!V<71H{==iCJ5SD zat2piE+9crJrlB)ej-*6I3mS+lf!B7YUj*@)T1*_s+1f+QMRGF2ojezS=u(2sNseg zl`Yc{B;mw+NYX{Q=+Z4wCc5*aK*!ZGQRt4B~DtcUd@?4@VN3hYdI`XcD5E77wr&IuUZ zh1=RAETzgxJ{oc^W*jx=#Fm`kQn_-b`T3e~FbQLjlMZCJB6YLk1rmZ> zr1h@HYFBarm)|PtN_I->04dhy>yC3RyI@}Zk&Q_XbQ1SkmW8IsF$Rq zmX^Y#A%UR0JT5*5>nPWckP%$rzWNz%+!zTyeaGdcC8r-o8AgExsK;ym1l~0@Rp7LBL(2 zP#vfMHHJ^9Ww_UDXiZH`93xpO>vopFz;gA-#+sc)IVl>7PGwdWRs57eXcjc8dNh0< zcCe2IGzc1wly#(>2m1!U0B*;(ta?Kd+ZyBk#vdLX0i)|@tL#M#B5I8gPmG`}RCoZe zap)Q2A2xm{`M+Cx zq$Qqg`&v~+^8t8;#A|MhY?G}Yw>?wcaRnV^1U&5ZnwLtoHaaLzv zR)HQB6W+56I`L(;9!kxegLV_obRDlf2NpN1$Z$;R9Gl>qT}PzT{8wFTy%c)b0zwX; z)6Z7mXFZ;j3tG~k6mmORI<~dWquy>>_-djpRRBLXq?@v8xfP?U3N2uQuqU^QkqrbB z;S~f1;D+E;s1@L0Rww}TZZKzR1{>#pO^|RNotF54*#uuH1;77D?#_AUoN09RF4BBvtA9R$upPAqa54Z&>=%xB83q;uPl5!$$Z% z!WREV|9IszHW9AyKlV3=nnP_L5Agr=H%il6^~bYPSoGiJuMWIa`6jv|{F(n@;H_Gu zg1rF%`A69|`NoO%jTZcjP-9tx7XbS@di)gOCkU@1;0sf{l^DJV7+#AE1&X1}F)VY2 ziS9<=CN603PCJ7p%whI6f=3g!?jn2}!1deRZg>)o?Iier2hC#DV!c1WPt;x^4qjzM z2lJ*{)amo=chCXJ?XxcPOD9tsVV^&Gz2#cVr+$2>e-iNdtG7jSJG!SQx7(AG_VuW{ zJ?j3Rp`Bnu|9?c!zAk+?n7%()YPToF$4?#NS5N)2?#ScdVexn9aj@xeu<>!Qh0hQ6 tCr16!h~&hd`P3&`AD<6Kq}J`YFIaVB{r7>U4}y``Mn4Mpg3W@U{{oq?u7dyo diff --git a/cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc b/cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60a6ecdb747cdf52eefa8bf26d97b0fe890abd96 GIT binary patch literal 3012 zcmbVOO>7&-6`o!GEJca>SL7IhwT^7bY$~!Sl-B*X97&1lR#6yOX#qyWaCbt3!dFaw(kJXnO1$N~BfAa?uPh zZ#Zw>_ukt#^WNht-HAAY@vDFQ=Gq^k2>o4cS|fb;_CxB-hFg^DPMuumW`mmzN4WEs3zwKIVMYPS)PBDR!jgxh}B;^<0yY z0*Kp^cm?TtCa+i}v*d9hD?S&D;Igc^cm-1j=N%&R%v&a|=~JIbaEu5uUp0L~SnZ0S zW#UzYvJ^d^o7~J`^Juo0{(EeskRxlD+U9lAo~3KbB% zY~FHsHZO%JK^L5Gi-pz5^*5yx94EEx^Lf~<)^vxBfY47BAy z%LUpBSJ^aXsu6rIUGmB#UB-f^;bo_mzviY~BGN@~!V|j_V)PA@mtATU7to!-^-7ps4G!_<1y$!|8)%pFq+r!YbbAZlx<(Y zVms`Ca`Zt$T0|MQ@DUfnO?(1%*0rj+iVGDH zY8X^~x&fkSl51DvAiNA^JLo5CKG-C7 z&Y@tuYJs0!D)Tvru28a?4AFlnC7&tsZy?FoCx3*Heex$o{tSdpsH2o3Ur|H_h&~3A z)PpdGZV`l_Lnh0tMb__X-vKPAZ9txOKG9X(y$$X$p^uaI@*tQ zZ-`r6^{M?t?`Gefp{=3KwO@?v^qp_?oxeA=ofxan?sg?^&)l5Z9KZY3?XENRncdFr z+ZS(M+{zkcuX_R#o;eZL+0Xz0W0$1iUuzw>dWF*vi`Ia{CJH@ep44Y@h9 zW1N0qoZjm>vv0)LYd30}=XQ+Y2gdMT&&cOE=RU_7-!&4O13&%AUe5*fYviF}?)5xn zjeWP>zy&LasPVN||9vG+Y&Njy8AHFo-c>BkVcb6W%z0|Pm9jkQTDmAFLZTXK7 zs%X>t6G$Og_IgJ5I$vZe*^`}tk;~bhmCFS>gBNOfWgq^W1lk-^^?^bALF`M4Kx;$n z%ZjLy-;D1^sHx;I^xrOfHgt#YvRC08f{ys_fjkOnnzo1D{447CTP%J%{_{9EGx)j= zbWXpbf2Nin>YpAgQ2#sT^?GDC8e6~i!L>%;a-;0+l&{|}U*C?hdZ^9MHL$gV$$d<= zqlF`U32NWhrj*+EwbztdwSZb@W-8%lCk|Gtt`3rNPU==JbF QqY*9fq<2_5-TcD;05PP&9{>OV literal 0 HcmV?d00001 diff --git a/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc b/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc index 6cb726170c56111b41a19a81529ccaf181d37f73..dea17073bd9cb4fe5887fa5ae2bd3a05c79f7a75 100644 GIT binary patch delta 305 zcmZoOpJB$Q&Bx2d00dLtuFLwzJ(2G(qvT{B#*dnrdC57YDVcfc3QBpYsVT+rMX6<( zspU!vY57G8B^jv-X^A;GNr}nXn|qitm=pz4%Mx?+i%Rq2K@7cu%3B=CIf=!^nQ4_p z;y~m1CkwD{VOzz?z>vr^`5=?R=2xt|OpH4x|6pGsp zjzV!perZmMLRw~toxG^$x@?{z{F{+Y3TQ*I?B=PWe2iQRSXS^|Q82q9EV+4^ zm>{#KI?G`lsX%p>Bhs>gI!wn^1cRA5j#~=`b8ww8lnUnMI4da;EW~!!j15ShGZ6?D pXF1Qw1|%=)a0fH6UNpB0vSz(x%^J)wN$LnIJ0r_?fypPO`~hemfcyXe diff --git a/cmd/needle-sidecar/eval/golden.jsonl b/cmd/needle-sidecar/eval/golden.jsonl index d6ee971..f9da265 100644 --- a/cmd/needle-sidecar/eval/golden.jsonl +++ b/cmd/needle-sidecar/eval/golden.jsonl @@ -1,40 +1,40 @@ -{"id": "ng-404-root", "src": "clickhouse", "expected": "log_404", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"} -{"id": "ng-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"} -{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"} -{"id": "orch-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"} -{"id": "orch-401-chat", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"} -{"id": "ng-401-manifest", "src": "clickhouse", "expected": "auth_failure", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"} -{"id": "ng-503-store", "src": "clickhouse", "expected": "escalate_outage", "note": "upstream 503, real user-facing route", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"} -{"id": "ng-500-root", "src": "clickhouse", "expected": "escalate_outage", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"} -{"id": "ng-500-wp", "src": "clickhouse", "expected": "security_alert", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"} -{"id": "ng-500-blog", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"} -{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"} -{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"} -{"id": "ng-404-env", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"} -{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"} -{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "security_alert", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"} -{"id": "ng-404-commit", "src": "clickhouse", "expected": "log_404", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"} -{"id": "ng-404-vite", "src": "clickhouse", "expected": "log_404", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"} -{"id": "ng-404-blob", "src": "clickhouse", "expected": "log_404", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"} -{"id": "ng-499-repos", "src": "clickhouse", "expected": "none", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"} -{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"} -{"id": "ng-308-blog", "src": "clickhouse", "expected": "none", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"} -{"id": "ng-200-speed", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"} -{"id": "ng-200-matrix", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"} -{"id": "ng-200-overview", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"} -{"id": "sched-readyz", "src": "clickhouse", "expected": "none", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"} -{"id": "cart-ping", "src": "clickhouse", "expected": "none", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"} -{"id": "rag-request", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"} -{"id": "orch-session-load", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"} -{"id": "worker-dialerr", "src": "clickhouse", "expected": "none", "note": "dependency down; arguably escalate_outage - decide", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/...: connection refused"} -{"id": "worker-eof", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"} -{"id": "ledger-skip", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"} -{"id": "web-cls", "src": "clickhouse", "expected": "record_vital", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls "} -{"id": "syn-js-1", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"} -{"id": "syn-js-2", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/"} -{"id": "syn-vital-1", "src": "synthetic", "expected": "record_vital", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"} -{"id": "syn-db-1", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after ms | Message: context deadline exceeded querying orders"} -{"id": "syn-db-2", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"} -{"id": "syn-db-3", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"} -{"id": "syn-oom", "src": "synthetic", "expected": "none", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"} -{"id": "syn-retry", "src": "synthetic", "expected": "none", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"} +{"id": "ng-404-root", "src": "clickhouse", "expected": "stale_reference", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"} +{"id": "ng-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"} +{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"} +{"id": "orch-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"} +{"id": "orch-401-chat", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"} +{"id": "ng-401-manifest", "src": "clickhouse", "expected": "client_credential_issue", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"} +{"id": "ng-503-store", "src": "clickhouse", "expected": "dependency_unreachable", "note": "nginx 503 burst: upstream down is the likeliest cause", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"} +{"id": "ng-500-root", "src": "clickhouse", "expected": "service_defect", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"} +{"id": "ng-500-wp", "src": "clickhouse", "expected": "scanner_probe", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"} +{"id": "ng-500-blog", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"} +{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"} +{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"} +{"id": "ng-404-env", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"} +{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"} +{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "scanner_probe", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"} +{"id": "ng-404-commit", "src": "clickhouse", "expected": "stale_reference", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"} +{"id": "ng-404-vite", "src": "clickhouse", "expected": "stale_reference", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"} +{"id": "ng-404-blob", "src": "clickhouse", "expected": "stale_reference", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"} +{"id": "ng-499-repos", "src": "clickhouse", "expected": "expected_lifecycle", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"} +{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"} +{"id": "ng-308-blog", "src": "clickhouse", "expected": "expected_lifecycle", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"} +{"id": "ng-200-speed", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"} +{"id": "ng-200-matrix", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"} +{"id": "ng-200-overview", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"} +{"id": "sched-readyz", "src": "clickhouse", "expected": "expected_lifecycle", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"} +{"id": "cart-ping", "src": "clickhouse", "expected": "expected_lifecycle", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"} +{"id": "rag-request", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"} +{"id": "orch-session-load", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"} +{"id": "worker-dialerr", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/...: connection refused"} +{"id": "worker-eof", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"} +{"id": "ledger-skip", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"} +{"id": "web-cls", "src": "clickhouse", "expected": "expected_lifecycle", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls "} +{"id": "syn-js-1", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"} +{"id": "syn-js-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/"} +{"id": "syn-vital-1", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"} +{"id": "syn-db-1", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after ms | Message: context deadline exceeded querying orders"} +{"id": "syn-db-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"} +{"id": "syn-db-3", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"} +{"id": "syn-oom", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"} +{"id": "syn-retry", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"} diff --git a/cmd/needle-sidecar/eval/holdout.jsonl b/cmd/needle-sidecar/eval/holdout.jsonl new file mode 100644 index 0000000..9ae830f --- /dev/null +++ b/cmd/needle-sidecar/eval/holdout.jsonl @@ -0,0 +1,60 @@ +{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e: accepted_remote=\u003cnum\u003e kept_local=\u003cnum\u003e legacy_no_stamp=\u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"service_defect","rule":"status-5xx"} +{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"} +{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"} +{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"dependency_unreachable","rule":"status-5xx-upstream"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e taking ownership of \u003cnum\u003e ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} diff --git a/cmd/needle-sidecar/eval/rules.py b/cmd/needle-sidecar/eval/rules.py new file mode 100644 index 0000000..9ea4a1c --- /dev/null +++ b/cmd/needle-sidecar/eval/rules.py @@ -0,0 +1,42 @@ +"""Deterministic baseline for the cause categories. Needle has to beat this.""" +import re + +PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I) +DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I) +CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I) +BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I) +LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I) + + +def fields(text): + d = {} + for part in text.split(" | "): + k, _, v = part.partition(": ") + d[k.lower()] = v + return d + + +def classify(text): + f = fields(text) + status = int(f.get("status") or 0) + route = f.get("route", "") + blob = " ".join([f.get("title", ""), f.get("message", ""), route]) + if PROBE.search(route): + return "scanner_probe" + if BUG.search(blob): + return "service_defect" + if CAP.search(blob): + return "capacity_saturation" + if DEP.search(blob): + return "dependency_unreachable" + if status == 401: + return "client_credential_issue" + if status in (502, 503, 504): + return "dependency_unreachable" + if status >= 500: + return "service_defect" + if status == 404: + return "stale_reference" + if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")): + return "expected_lifecycle" + return "needs_review" diff --git a/cmd/needle-sidecar/eval/run_eval.py b/cmd/needle-sidecar/eval/run_eval.py index 67d0d06..b967013 100755 --- a/cmd/needle-sidecar/eval/run_eval.py +++ b/cmd/needle-sidecar/eval/run_eval.py @@ -4,8 +4,7 @@ Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json] Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name -or "none" when no tool should fit. The sidecar has no fallback tool, so "none" -rows measure how often it force-fits (see the confidence column). +including "needs_review" for the fallback. """ import argparse, json, os, sys, time, urllib.request from collections import Counter, defaultdict @@ -29,7 +28,7 @@ def main(): ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl")) ap.add_argument("--json", help="write per-row results here") ap.add_argument("--min-conf", type=float, default=0.0, - help="predictions below this confidence count as 'none'") + help="predictions below this confidence count as needs_review") a = ap.parse_args() rows = [json.loads(l) for l in open(a.golden) if l.strip()] @@ -41,10 +40,10 @@ def main(): except Exception as e: print(f"sidecar error on {row['id']}: {e}", file=sys.stderr) sys.exit(2) - pred = out.get("action", "unclassified") + pred = out.get("action", "needs_review") conf = out.get("confidence", 0.0) - if conf < a.min_conf or pred == "unclassified": - pred = "none" + if conf < a.min_conf: + pred = "needs_review" lat.append(out.get("latency_ms", 0.0)) confusion[(row["expected"], pred)] += 1 results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]}) diff --git a/cmd/needle-sidecar/server.py b/cmd/needle-sidecar/server.py index 48e7d1c..7da5bff 100755 --- a/cmd/needle-sidecar/server.py +++ b/cmd/needle-sidecar/server.py @@ -16,46 +16,49 @@ import needle PORT = int(os.environ.get("NEEDLE_PORT", 5005)) HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0") -# --- Define Triage Tools for Needle 3 --- +# --- Cause-level triage tools --- +# These deliberately do not restate what the event already carries (status +# code, is_error, severity). Each one is a judgement about *why* it happened +# or *who should care*, which is what an alert reader needs. @needle.tool -def report_client_javascript_crash(service: str, message: str, url: str): - """Report client-side JavaScript crash or error logged by browser RUM.""" - return {"action": "client_js_crash", "service": service, "message": message, "url": url} +def report_scanner_probe(service: str, path: str, status_code: int): + """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502.""" + return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code} @needle.tool -def alert_vulnerability_probe(scanner: str, target: str, status_code: int): - """Alert on malicious vulnerability scans or exploit crawlers.""" - return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code} +def report_dependency_unreachable(service: str, dependency: str, error_message: str): + """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable.""" + return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message} @needle.tool -def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int): - """Record RUM performance metrics like TTFB or FCP.""" - return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms} +def report_client_credential_issue(service: str, route: str, reason: str): + """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service.""" + return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason} @needle.tool -def log_missing_resource_404(path: str, host: str): - """Log benign 404 Not Found requests for missing pages or git commits.""" - return {"action": "log_404", "path": path, "host": host} +def report_stale_reference(service: str, path: str): + """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request.""" + return {"action": "stale_reference", "service": service, "path": path} @needle.tool -def report_database_error(service: str, table: str, error_message: str): - """Report database query errors, deadlocks, or connection pool timeouts.""" - return {"action": "database_error", "service": service, "table": table, "error": error_message} +def report_service_defect(service: str, error_message: str): + """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route.""" + return {"action": "service_defect", "service": service, "error": error_message} @needle.tool -def report_auth_failure(service: str, user_or_token: str, reason: str): - """Report authentication/authorization failure or invalid credentials.""" - return {"action": "auth_failure", "service": service, "user_or_token": user_or_token, "reason": reason} +def report_capacity_saturation(service: str, resource: str, error_message: str): + """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full.""" + return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message} @needle.tool -def escalate_backend_outage(service: str, error_message: str): - """Escalate severe backend 5xx crash or unhandled panic to on-call.""" - return {"action": "escalate_outage", "service": service, "error": error_message} +def report_expected_lifecycle(service: str, what: str): + """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals.""" + return {"action": "expected_lifecycle", "service": service, "what": what} @needle.tool -def report_unclassified(service: str, summary: str): - """Fallback when the event matches none of the other categories: retries, cancellations, restarts, OOM, health checks, or anything unclear.""" - return {"action": "unclassified", "service": service, "summary": summary} +def report_needs_review(service: str, summary: str): + """Fallback when none of the other categories clearly applies.""" + return {"action": "needs_review", "service": service, "summary": summary} # --- Redaction: nothing PII-bearing should reach the model or the cache --- _REDACTIONS = [ @@ -81,14 +84,14 @@ _stats = {"hits": 0, "misses": 0} print(f"Loading Needle 3 model (Cactus-Compute/needle3)...") t0 = time.perf_counter() agent = needle.Needle(tools=[ - report_client_javascript_crash, - alert_vulnerability_probe, - record_frontend_web_vital, - log_missing_resource_404, - report_database_error, - report_auth_failure, - escalate_backend_outage, - report_unclassified + report_scanner_probe, + report_dependency_unreachable, + report_client_credential_issue, + report_stale_reference, + report_service_defect, + report_capacity_saturation, + report_expected_lifecycle, + report_needs_review, ]) print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s") @@ -116,14 +119,8 @@ class NeedleHandler(BaseHTTPRequestHandler): "model": "Cactus-Compute/needle3", "cache": {**_stats, "size": len(_cache)}, "tools": [ - "client_js_crash", - "security_alert", - "record_vital", - "log_404", - "database_error", - "auth_failure", - "escalate_outage", - "unclassified" + "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference", + "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review" ] } self.wfile.write(json.dumps(resp).encode("utf-8")) @@ -193,7 +190,7 @@ class NeedleHandler(BaseHTTPRequestHandler): results = res.get("results", []) first_result = results[0] if results and isinstance(results[0], dict) else {} - action = first_result.get("action", "unclassified") + action = first_result.get("action", "needs_review") calls = res.get("function_calls", []) tool_name = calls[0].get("name") if calls else "triage_tool" diff --git a/cmd/normalized-backfill/main.go b/cmd/normalized-backfill/main.go index 2445467..dd01d78 100644 --- a/cmd/normalized-backfill/main.go +++ b/cmd/normalized-backfill/main.go @@ -12,6 +12,7 @@ import ( "time" "git.tornberg.me/go-otel/internal/normalization" + "git.tornberg.me/go-otel/internal/storage" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" ) @@ -50,7 +51,7 @@ func main() { log.Fatal(err) } if *write { - if err := createSchema(ctx, conn, *database); err != nil { + if err := storage.EnsureNormalizedSchema(ctx, conn, *database); err != nil { log.Fatal(err) } } @@ -82,7 +83,7 @@ func main() { rawCount, eventCount := 0, 0 var batch driver.Batch if *write { - batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database)) + batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database)) if err != nil { log.Fatal(err) } @@ -115,12 +116,12 @@ func main() { } counts[key].Count++ if *write { - if err := appendEvent(batch, event); err != nil { + if err := storage.AppendNormalizedEvent(batch, event); err != nil { log.Fatal(err) } if eventCount%*batchSize == 0 { flush() - batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database)) + batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database)) if err != nil { log.Fatal(err) } @@ -146,70 +147,6 @@ func main() { } } -func createSchema(ctx context.Context, conn clickhouse.Conn, database string) error { - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events ( - event_id String, - entry_type LowCardinality(String), - event_index UInt32, - ingest_timestamp DateTime64(9, 'UTC'), - timestamp DateTime64(9, 'UTC'), - service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String), - kind LowCardinality(String), source LowCardinality(String), is_error UInt8, - status_class LowCardinality(String), status_code UInt16, duration_ms Float64, - route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String, - metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64, - exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String, - is_retry UInt8, is_timeout UInt8, is_cancellation UInt8, - exact_signature String, pattern_signature String, version UInt8 DEFAULT 1, - method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '', - origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT '' - ) ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(timestamp) - ORDER BY (service, kind, pattern_signature, timestamp, event_id) - TTL toDateTime(timestamp) + INTERVAL 180 DAY`, database) - if err := conn.Exec(ctx, query); err != nil { - return err - } - // Existing developer tables predate the explicit model flags. These - // additive migrations preserve historical rows with safe zero defaults. - for _, statement := range []string{ - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS issue_fingerprint String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS method String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS host String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS origin_host String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS referer_host String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_site String DEFAULT ''", database), - fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_mode String DEFAULT ''", database), - } { - if err := conn.Exec(ctx, statement); err != nil { - return err - } - } - return nil -} - -func appendEvent(batch driver.Batch, event normalization.Event) error { - return batch.Append( - event.EventID, event.EntryType, event.EventIndex, time.Unix(0, event.IngestTimestamp), time.Unix(0, event.Timestamp), - event.Service, event.Release, event.Environment, event.Kind, event.Source, boolUint8(event.IsError), - event.StatusClass, event.StatusCode, event.DurationMS, event.Route, event.SpanName, event.MetricName, event.MetricType, event.MetricUnit, - event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax, - event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint, - boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature, - uint8(1), event.Method, event.Host, event.UserAgent, event.OriginHost, event.RefererHost, event.FetchSite, event.FetchMode, - ) -} - -func boolUint8(value bool) uint8 { - if value { - return 1 - } - return 0 -} func envOr(key, fallback string) string { if value := os.Getenv(key); value != "" { return value diff --git a/cmd/server/main.go b/cmd/server/main.go index f5677d8..ba06d30 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -11,6 +11,7 @@ import ( "git.tornberg.me/go-otel/internal/auth" "git.tornberg.me/go-otel/internal/collector" + "git.tornberg.me/go-otel/internal/blocker" "git.tornberg.me/go-otel/internal/honeypot" kube "git.tornberg.me/go-otel/internal/kubernetes" "git.tornberg.me/go-otel/internal/prometheus" @@ -235,6 +236,12 @@ func main() { wsServer := websocket.New(coll) honeypotTracker := honeypot.New(honeypot.ConfigFromEnv()) coll.SetHoneypotRecorder(honeypotTracker) + // Optional ingress blocker: off unless OTEL_BLOCKER_ENABLED=true, and + // dry-run unless OTEL_BLOCKER_DRY_RUN=false. + ipBlocker, err := blocker.Start(context.Background(), blocker.ConfigFromEnv(), honeypotTracker) + if err != nil { + log.Printf("ip blocker disabled: %v", err) + } // Update collector with wsServer coll.SetBroadcaster(wsServer) ingestKey := os.Getenv("OTEL_INGEST_KEY") @@ -293,6 +300,9 @@ func main() { uiMux.HandleFunc("POST /api/auth/token", oidcAuth.HandleTokenExchange) uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig) uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser))) + if ipBlocker != nil { + uiMux.Handle("GET /api/blocker/status", oidcAuth.RequireAuth(http.HandlerFunc(ipBlocker.HandleStatus))) + } // Source-map uploads are intentionally public so the release build script // can upload maps without an OIDC session or bearer token. uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap) diff --git a/cmd/spike-alerts/main.go b/cmd/spike-alerts/main.go index efe4c8b..5e5d049 100644 --- a/cmd/spike-alerts/main.go +++ b/cmd/spike-alerts/main.go @@ -18,6 +18,7 @@ import ( "github.com/ClickHouse/clickhouse-go/v2" "git.tornberg.me/go-otel/internal/features" + "git.tornberg.me/go-otel/internal/triage" ) func main() { @@ -33,7 +34,8 @@ func main() { minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert") multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline") cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles") - needle := flag.String("needle", os.Getenv("OTEL_NEEDLE_ADDR"), "Needle sidecar URL; empty skips classification") + needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it") + showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)") asJSON := flag.Bool("json", false, "emit JSON lines instead of text") flag.Parse() if *since == "" || *until == "" { @@ -55,9 +57,10 @@ func main() { cache := map[string]classification{} first := start.Add(time.Duration(*baselineWindows) * *window) suppress := features.NewSuppressor(*cooldown) + muted := map[triage.Category]int{} alerts := 0 for at := first.Add(*window); !at.After(end); at = at.Add(*step) { - for _, spike := range suppress.Filter(features.DetectSpikes(events, at, cfg)) { + for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) { alerts++ var cls []classification for _, p := range spike.Patterns { @@ -79,6 +82,9 @@ func main() { printSpike(spike, cls) } } + for c, n := range muted { + log.Printf("muted %d pattern-window(s) as %s", n, c) + } log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339)) } @@ -89,6 +95,35 @@ type classification struct { Err string `json:"error,omitempty"` } +func triageOf(e features.Event) triage.Result { + return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message, + IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry}) +} + +// triageFilter keeps only actionable patterns (unless showMuted), recomputing +// each spike's excess so a probe storm cannot page anyone by itself. +func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike { + var out []features.Spike + for _, sp := range spikes { + var kept []features.PatternDelta + sp.Excess = 0 + for _, p := range sp.Patterns { + c := triageOf(p.Example).Category + if !showMuted && !c.Actionable() { + muted[c]++ + continue + } + kept = append(kept, p) + sp.Excess += p.Excess + } + if len(kept) > 0 { + sp.Patterns = kept + out = append(out, sp) + } + } + return out +} + func classify(base, text string) classification { body, _ := json.Marshal(map[string]string{"text": text}) resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body)) @@ -127,9 +162,9 @@ func printSpike(s features.Spike, cls []classification) { if p.Novel { tag = "NEW" } - what := "-" + what := string(triageOf(p.Example).Category) if cls[i].Action != "" { - what = fmt.Sprintf("%s (%.2f)", cls[i].Action, cls[i].Confidence) + what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence) } fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example)) } @@ -144,7 +179,7 @@ func load(addr, database, user, password string, start, end time.Time) []feature } defer conn.Close() // Errors only, and only columns present in every deployed schema version. - rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature + rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry FROM %s.normalized_events FINAL WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC') ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end))) @@ -156,10 +191,12 @@ func load(addr, database, user, password string, start, end time.Time) []feature for rows.Next() { e := features.Event{IsError: true} var status uint16 - if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature); err != nil { + var timeout, cancel, retry uint8 + if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil { log.Fatal(err) } e.StatusCode = int(status) + e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1 out = append(out, e) } if err := rows.Err(); err != nil { diff --git a/cmd/triage-eval/main.go b/cmd/triage-eval/main.go new file mode 100644 index 0000000..ed40fde --- /dev/null +++ b/cmd/triage-eval/main.go @@ -0,0 +1,98 @@ +// triage-eval applies the rules classifier to a JSONL file of patterns. Rows +// with an "expected" category are scored; the rest are written out with the +// prediction so they can be reviewed and labelled by hand. +// +// triage-eval -input holdout.jsonl # score labelled rows +// triage-eval -input raw.jsonl -label-out holdout.jsonl # prepare for labelling +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "sort" + + "git.tornberg.me/go-otel/internal/triage" +) + +type row struct { + Service string `json:"service"` + Kind string `json:"kind"` + StatusCode int `json:"status_code"` + Route string `json:"route"` + Title string `json:"title"` + Message string `json:"message"` + IsTimeout uint8 `json:"is_timeout"` + IsCancellation uint8 `json:"is_cancellation"` + IsRetry uint8 `json:"is_retry"` + Count uint64 `json:"n"` + Expected string `json:"expected"` + Predicted string `json:"predicted"` + Rule string `json:"rule"` +} + +func main() { + input := flag.String("input", "", "JSONL input") + labelOut := flag.String("label-out", "", "write rows with predictions here for hand labelling") + flag.Parse() + f, err := os.Open(*input) + if err != nil { + log.Fatal(err) + } + defer f.Close() + var rows []row + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + var r row + if err := json.Unmarshal(sc.Bytes(), &r); err != nil { + log.Fatalf("bad row: %v", err) + } + res := triage.Classify(triage.Input{Service: r.Service, Kind: r.Kind, Status: r.StatusCode, Route: r.Route, Title: r.Title, Message: r.Message, + IsTimeout: r.IsTimeout == 1, IsCancellation: r.IsCancellation == 1, IsRetry: r.IsRetry == 1}) + r.Predicted, r.Rule = string(res.Category), res.Rule + rows = append(rows, r) + } + if *labelOut != "" { + out, err := os.Create(*labelOut) + if err != nil { + log.Fatal(err) + } + defer out.Close() + enc := json.NewEncoder(out) + for _, r := range rows { + _ = enc.Encode(r) + } + log.Printf("wrote %d rows to %s; fill in \"expected\" then re-run without -label-out", len(rows), *labelOut) + return + } + scored, correct := 0, 0 + confusion := map[[2]string]int{} + for _, r := range rows { + if r.Expected == "" { + continue + } + scored++ + if r.Expected == r.Predicted { + correct++ + } else { + confusion[[2]string{r.Expected, r.Predicted}]++ + fmt.Printf("miss expected=%-24s got=%-24s rule=%-18s %s %d %s %.80s\n", r.Expected, r.Predicted, r.Rule, r.Service, r.StatusCode, r.Route, r.Message) + } + } + if scored == 0 { + log.Fatal("no labelled rows (set \"expected\")") + } + fmt.Printf("\naccuracy %d/%d = %.0f%%\n", correct, scored, 100*float64(correct)/float64(scored)) + keys := make([][2]string, 0, len(confusion)) + for k := range confusion { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return confusion[keys[i]] > confusion[keys[j]] }) + for _, k := range keys { + fmt.Printf(" %-24s -> %-24s x%d\n", k[0], k[1], confusion[k]) + } +} diff --git a/docs/ML.md b/docs/ML.md index fefa71e..d3a5dda 100644 --- a/docs/ML.md +++ b/docs/ML.md @@ -1,5 +1,8 @@ # Error detection model +> Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they +> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins. + ## Current handoff (2026-09-18) The first iteration is intentionally explainable and deterministic. Raw OTLP diff --git a/docs/STATUS.md b/docs/STATUS.md new file mode 100644 index 0000000..edb1533 --- /dev/null +++ b/docs/STATUS.md @@ -0,0 +1,132 @@ +# Status: error triage, spike detection, IP blocking (2026-09-19) + +Pick-up notes. Nothing below is committed, deployed or applied to the cluster +unless stated. Greenfield, single user: radical changes are fine. + +## Access + +- ClickHouse (otel instance) is now a **LoadBalancer**: `monitoring/clickhouse` + at `10.10.3.45` (native 9000, HTTP 8123). Set in `k8s/clickhouse.yaml` and + patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded. +- Credentials are in `.env` (gitignored, mode 600). Load with + `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally. +- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly. + +## What exists now + +| Piece | Where | State | +| --- | --- | --- | +| Normalized rows written on ingest | `internal/storage/normalized.go`, `Append` in `clickhouse.go` | Code done, tests pass. **Running collector is still the old build**, so nothing new is written until deployed. | +| Live schema migration | `EnsureNormalizedSchema` (runs at collector start) | Applied once by hand via backfill; `issue_fingerprint` etc. now exist. | +| 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. | +| Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). | +| Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. | +| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. | +| Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. | +| Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. | +| Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. | +| Tracker | `internal/honeypot` | Now needs distinct paths (default 3), never tracks internal/reserved IPs, uses `triage.IsProbePath`. | +| Blocker | `internal/blocker` | Writes ingress-nginx `block-cidrs`. **Dry-run by default.** Tested with a fake client. Wired into `cmd/server`. | +| RBAC + env | `k8s/deployment.yaml` | Role limited to the one ConfigMap; blocker env set to enabled + dry-run. Not applied. | + +Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`, +`stale_reference`, `service_defect`, `capacity_saturation`, `expected_lifecycle`, +`needs_review`. They are cause-level verdicts, not restatements of status. + +## Findings worth remembering + +- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes). + Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`), + so it picks by position, not content. Confidence is uncorrelated with + correctness. Decision: **Needle stays dormant.** Intended later use: a gate that + decides whether a larger model should act on an issue (affects anyone? fixable by + a smart model?). Do not use it for categorization. +- **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing + the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl` + (from 2026-09-18, unseen); it still needs your `expected` labels. Known + probe-like paths that still land in `stale_reference`: `/ssl/server.key`, + `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`, + `/.bash_profile`, `/settings.py`, `/__env.js`. +- **The top error sources were noise:** 404 probes and a 401 poller + (`/api/repos/available`, `/api/controller/instances`, about 700 per hour, counted + twice: nginx and the orchestrator). +- **IP data:** 44 public IPs account for 63% of probe requests (12.5 h window). + **34% show a peer of `127.0.0.1`** because `net.peer.ip` is the TCP peer behind + the PROXY-protocol hop, so the tracker cannot see them. The ingress has + `enable-real-ip=true`, `forwarded-for-header=proxy_protocol`, + `proxy-real-ip-cidr=127.0.0.1/32`, so `block-cidrs` itself works on the real + client address. +- **Internal addresses are a hard stop** in both tracker and blocker + (`honeypot.Untrackable`): loopback, RFC1918, ULA, link-local, multicast, + unspecified, IPv4-mapped, CGNAT `100.64/10` (Tailscale), `192.0.0.0/24`, + `198.18/15`, `240/4`. The allowlist only adds exclusions. + +## Before enabling enforcement + +1. Deploy the new build (dry-run on). Watch `GET /api/blocker/status` for a day; + compare against `go run ./cmd/bad-actors -since ... -until ...`. +2. Put your own public IP(s) in `OTEL_BLOCKER_ALLOW`. A blocked IP gets 403, so + you would lock yourself out of the UI. Recovery: `kubectl edit cm + ingress-nginx-controller -n ingress-nginx` (remove entries; managed ones are + listed in the `otel-debug.k6n.net/blocked` annotation). +3. Then set `OTEL_BLOCKER_DRY_RUN=false`. +4. Do not `kubectl apply` the whole `k8s/deployment.yaml` blindly: the dry run + showed it would create a new `otel-sourcemaps` PVC not present in the cluster. + +## Next steps (suggested order) + +1. Deploy the collector so normalized rows stay current; verify with + `SELECT max(timestamp) FROM otel.normalized_events`. +2. Label `holdout.jsonl` (fill `expected`), run + `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix + the rule families that miss. Then re-export a fresh holdout so scoring stays + honest. +3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the + first window after data begins alerts on everything as NEW). +4. Real client IP in nginx spans, to close the 127.0.0.1 gap (inspect the + ingress OTel config first; not looked at yet). +5. Alert delivery: run `DetectSpikes` on a timer in the collector and post + somewhere (UI panel / ntfy / Slack). Category chips and `same / different / + noise` + "mute pattern" in the UI, storing labels. +6. Reduce 401-poller noise: mute list or per-pattern rate limit; + `client_credential_issue` is the caller's fault, likely a lower severity. +7. After 2+ weeks of data: time-of-day baseline, rollout proximity, then the + logistic regression / MLP from `docs/ML.md`; the rule detector is what they + must beat. + +## Commands + +```bash +set -a; . ./.env; set +a + +# replay the spike rule over a range (Needle off) +go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12 +# add -show-muted to see probes / stale / lifecycle patterns too + +# who would be blocked +go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z + +# score the rules against labelled rows +go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl + +# Needle eval (needs the sidecar running; venv was in scratch and is gone) +pip install cactus-needle # python 3.12 recommended +NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py +python cmd/needle-sidecar/eval/run_eval.py +``` + +Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows +through a `kubectl port-forward` hung and dropped. + +## Git state + +Everything is uncommitted. Files changed by this work: `internal/{storage, +normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts, +triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`, +`k8s/{clickhouse,deployment}.yaml`, `.gitignore`. + +**Not from this work; check before committing:** `AGENTS.md`, +`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` +(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files +under `cmd/needle-sidecar` show as modified; they should be untracked and +gitignored. diff --git a/internal/blocker/blocker.go b/internal/blocker/blocker.go new file mode 100644 index 0000000..fce7a0d --- /dev/null +++ b/internal/blocker/blocker.go @@ -0,0 +1,383 @@ +// Package blocker turns the honeypot's promoted-IP feed into ingress-nginx +// `block-cidrs` entries. +// +// Safety properties, all deliberate: +// - dry-run is the default; nothing is written unless DryRun is false; +// - it only manages entries it added itself (tracked in an annotation on the +// ConfigMap), so manually configured block-cidrs are preserved untouched; +// - loopback/private/link-local addresses and an operator allowlist are never +// blocked; +// - blocks expire, the set is capped, growth per tick is limited, and writes +// are rate-limited because every ConfigMap change reloads nginx. +package blocker + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "net/netip" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "git.tornberg.me/go-otel/internal/honeypot" +) + +const AnnotationManaged = "otel-debug.k6n.net/blocked" + +type Source interface { + PromotedIPs() []honeypot.IP +} + +type Config struct { + Enabled bool + DryRun bool + Namespace string + ConfigMap string + Key string + Interval time.Duration // how often the feed is evaluated + MinWrite time.Duration // minimum gap between ConfigMap writes + BlockTTL time.Duration // how long a block lasts after last sighting + MinProbes int // per-window probe hits required to block + MinPaths int // distinct probe paths required to block + BroadProbes int // hits+paths that block regardless of MinProbes + BroadPaths int + MaxEntries int // cap on entries this blocker manages + MaxNewPerTick int // limit on newly added entries per evaluation + Allow []netip.Prefix +} + +func DefaultConfig() Config { + return Config{ + DryRun: true, Namespace: "ingress-nginx", ConfigMap: "ingress-nginx-controller", Key: "block-cidrs", + Interval: time.Minute, MinWrite: 5 * time.Minute, BlockTTL: 24 * time.Hour, + MinProbes: 10, MinPaths: 3, BroadProbes: 100, BroadPaths: 50, MaxEntries: 500, MaxNewPerTick: 20, + } +} + +// ConfigFromEnv reads OTEL_BLOCKER_*. Enabled defaults to false and DryRun to +// true; both must be changed on purpose. +func ConfigFromEnv() Config { + c := DefaultConfig() + c.Enabled = strings.EqualFold(os.Getenv("OTEL_BLOCKER_ENABLED"), "true") + if v := os.Getenv("OTEL_BLOCKER_DRY_RUN"); v != "" { + c.DryRun = !strings.EqualFold(v, "false") + } + str := func(env string, dst *string) { + if v := os.Getenv(env); v != "" { + *dst = v + } + } + num := func(env string, dst *int) { + if n, err := strconv.Atoi(os.Getenv(env)); err == nil && n > 0 { + *dst = n + } + } + dur := func(env string, dst *time.Duration) { + if d, err := time.ParseDuration(os.Getenv(env)); err == nil && d > 0 { + *dst = d + } + } + str("OTEL_BLOCKER_NAMESPACE", &c.Namespace) + str("OTEL_BLOCKER_CONFIGMAP", &c.ConfigMap) + str("OTEL_BLOCKER_KEY", &c.Key) + num("OTEL_BLOCKER_MIN_PROBES", &c.MinProbes) + num("OTEL_BLOCKER_MIN_PATHS", &c.MinPaths) + num("OTEL_BLOCKER_MAX_ENTRIES", &c.MaxEntries) + dur("OTEL_BLOCKER_INTERVAL", &c.Interval) + dur("OTEL_BLOCKER_MIN_WRITE", &c.MinWrite) + dur("OTEL_BLOCKER_TTL", &c.BlockTTL) + for _, item := range strings.Split(os.Getenv("OTEL_BLOCKER_ALLOW"), ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if p, err := netip.ParsePrefix(item); err == nil { + c.Allow = append(c.Allow, p) + } else if a, err := netip.ParseAddr(item); err == nil { + c.Allow = append(c.Allow, netip.PrefixFrom(a, a.BitLen())) + } else { + log.Printf("blocker: ignoring invalid OTEL_BLOCKER_ALLOW entry %q", item) + } + } + return c +} + +// Start builds a client (in-cluster, or KUBECONFIG locally) and runs the +// blocker in the background. It returns nil, nil when disabled. +func Start(ctx context.Context, cfg Config, source Source) (*Blocker, error) { + if !cfg.Enabled { + return nil, nil + } + rc, err := rest.InClusterConfig() + if err != nil { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, herr := os.UserHomeDir() + if herr != nil { + return nil, herr + } + kubeconfig = home + "/.kube/config" + } + if rc, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil { + return nil, fmt.Errorf("blocker kubernetes config: %w", err) + } + } + client, err := kubernetes.NewForConfig(rc) + if err != nil { + return nil, fmt.Errorf("blocker kubernetes client: %w", err) + } + b := New(cfg, client, source) + go b.Run(ctx) + return b, nil +} + +type Blocker struct { + cfg Config + client kubernetes.Interface + source Source + now func() time.Time + + mu sync.Mutex + lastWrite time.Time + lastErr string + last Status +} + +// Status is a snapshot for the UI and for tests. +type Status struct { + Enabled bool `json:"enabled"` + DryRun bool `json:"dryRun"` + Evaluated time.Time `json:"evaluated"` + Candidates []string `json:"candidates"` + Managed map[string]int64 `json:"managed"` // ip -> unix expiry + Wrote bool `json:"wrote"` + LastWrite time.Time `json:"lastWrite"` + Error string `json:"error,omitempty"` +} + +func New(cfg Config, client kubernetes.Interface, source Source) *Blocker { + d := DefaultConfig() + if cfg.Interval <= 0 { + cfg.Interval = d.Interval + } + if cfg.MinWrite <= 0 { + cfg.MinWrite = d.MinWrite + } + if cfg.BlockTTL <= 0 { + cfg.BlockTTL = d.BlockTTL + } + if cfg.MaxEntries <= 0 { + cfg.MaxEntries = d.MaxEntries + } + if cfg.MaxNewPerTick <= 0 { + cfg.MaxNewPerTick = d.MaxNewPerTick + } + if cfg.MinProbes <= 0 { + cfg.MinProbes = d.MinProbes + } + if cfg.MinPaths <= 0 { + cfg.MinPaths = d.MinPaths + } + if cfg.BroadProbes <= 0 { + cfg.BroadProbes = d.BroadProbes + } + if cfg.BroadPaths <= 0 { + cfg.BroadPaths = d.BroadPaths + } + return &Blocker{cfg: cfg, client: client, source: source, now: time.Now} +} + +// Run evaluates the feed until ctx is cancelled. +func (b *Blocker) Run(ctx context.Context) { + log.Printf("blocker: started (dryRun=%t, %s/%s key=%s)", b.cfg.DryRun, b.cfg.Namespace, b.cfg.ConfigMap, b.cfg.Key) + t := time.NewTicker(b.cfg.Interval) + defer t.Stop() + for { + if err := b.Tick(ctx); err != nil { + log.Printf("blocker: %v", err) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// Blockable applies the policy to one promoted IP. +func (b *Blocker) Blockable(ip honeypot.IP) bool { + if honeypot.Untrackable(ip.Address) { + return false + } + addr, err := netip.ParseAddr(ip.Address) + if err != nil { + return false + } + for _, p := range b.cfg.Allow { + if p.Contains(addr) { + return false + } + } + return (ip.Count >= b.cfg.MinProbes && ip.Paths >= b.cfg.MinPaths) || + (ip.Count >= b.cfg.BroadProbes && ip.Paths >= b.cfg.BroadPaths) +} + +// Tick runs one evaluation. It is exported so tests and a manual trigger can +// drive it without waiting on the timer. +func (b *Blocker) Tick(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + now := b.now() + status := Status{Enabled: true, DryRun: b.cfg.DryRun, Evaluated: now, Managed: map[string]int64{}, LastWrite: b.lastWrite} + defer func() { b.last = status }() + + cms := b.client.CoreV1().ConfigMaps(b.cfg.Namespace) + cm, err := cms.Get(ctx, b.cfg.ConfigMap, metav1.GetOptions{}) + if err != nil { + b.lastErr = err.Error() + status.Error = b.lastErr + return fmt.Errorf("get configmap: %w", err) + } + + previous := map[string]int64{} + if raw := cm.Annotations[AnnotationManaged]; raw != "" { + if err := json.Unmarshal([]byte(raw), &previous); err != nil { + previous = map[string]int64{} // unreadable state: treat as empty rather than guess + } + } + managed := map[string]int64{} + for ip, exp := range previous { + if exp > now.Unix() { + managed[ip] = exp + } + } + + added := 0 + for _, cand := range b.source.PromotedIPs() { + if !b.Blockable(cand) { + continue + } + status.Candidates = append(status.Candidates, cand.Address) + _, exists := managed[cand.Address] + if !exists { + if added >= b.cfg.MaxNewPerTick { + continue + } + added++ + } + managed[cand.Address] = now.Add(b.cfg.BlockTTL).Unix() + } + sort.Strings(status.Candidates) + for len(managed) > b.cfg.MaxEntries { // drop the soonest-expiring first + var oldest string + for ip, exp := range managed { + if oldest == "" || exp < managed[oldest] || (exp == managed[oldest] && ip < oldest) { + oldest = ip + } + } + delete(managed, oldest) + } + status.Managed = managed + + current := splitCIDRs(cm.Data[b.cfg.Key]) + manual := make([]string, 0, len(current)) + for _, entry := range current { + if _, wasOurs := previous[entry]; !wasOurs { + manual = append(manual, entry) + } + } + desired := append([]string(nil), manual...) + for ip := range managed { + desired = append(desired, ip) + } + desired = dedupSorted(desired) + changed := strings.Join(desired, ",") != strings.Join(dedupSorted(current), ",") + + if !changed { + return nil + } + if b.cfg.DryRun { + log.Printf("blocker[dry-run]: would set %s to %d entries (%d managed): %v", b.cfg.Key, len(desired), len(managed), managedList(managed)) + return nil + } + if !b.lastWrite.IsZero() && now.Sub(b.lastWrite) < b.cfg.MinWrite { + return nil // rate-limited; the change is picked up on a later tick + } + state, _ := json.Marshal(managed) + if cm.Annotations == nil { + cm.Annotations = map[string]string{} + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Annotations[AnnotationManaged] = string(state) + cm.Data[b.cfg.Key] = strings.Join(desired, ",") + if len(desired) == 0 { + delete(cm.Data, b.cfg.Key) + } + if _, err := cms.Update(ctx, cm, metav1.UpdateOptions{}); err != nil { + b.lastErr = err.Error() + status.Error = b.lastErr + return fmt.Errorf("update configmap: %w", err) + } + b.lastWrite, b.lastErr = now, "" + status.Wrote, status.LastWrite = true, now + log.Printf("blocker: updated %s: %d entries (%d managed)", b.cfg.Key, len(desired), len(managed)) + return nil +} + +// HandleStatus serves the last evaluation. Register it behind authentication. +func (b *Blocker) HandleStatus(w http.ResponseWriter, r *http.Request) { + b.mu.Lock() + s := b.last + b.mu.Unlock() + if s.Evaluated.IsZero() { + s = Status{Enabled: true, DryRun: b.cfg.DryRun, Managed: map[string]int64{}} + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(s) +} + +func splitCIDRs(v string) []string { + var out []string + for _, p := range strings.Split(v, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +func dedupSorted(in []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(in)) + for _, v := range in { + if _, ok := seen[v]; !ok { + seen[v] = struct{}{} + out = append(out, v) + } + } + sort.Strings(out) + return out +} + +func managedList(m map[string]int64) []string { + out := make([]string, 0, len(m)) + for ip := range m { + out = append(out, ip) + } + sort.Strings(out) + return out +} diff --git a/internal/blocker/blocker_test.go b/internal/blocker/blocker_test.go new file mode 100644 index 0000000..1400a07 --- /dev/null +++ b/internal/blocker/blocker_test.go @@ -0,0 +1,153 @@ +package blocker + +import ( + "context" + "encoding/json" + "net/netip" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "git.tornberg.me/go-otel/internal/honeypot" +) + +type feed []honeypot.IP + +func (f feed) PromotedIPs() []honeypot.IP { return f } + +func setup(t *testing.T, cfg Config, data map[string]string, src Source) (*Blocker, *fake.Clientset, *time.Time) { + t.Helper() + client := fake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ingress-nginx", Name: "ingress-nginx-controller"}, + Data: data, + }) + b := New(cfg, client, src) + clock := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC) + b.now = func() time.Time { return clock } + return b, client, &clock +} + +func current(t *testing.T, c *fake.Clientset) *corev1.ConfigMap { + t.Helper() + cm, err := c.CoreV1().ConfigMaps("ingress-nginx").Get(context.Background(), "ingress-nginx-controller", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return cm +} + +func scanner(ip string) honeypot.IP { return honeypot.IP{Address: ip, Count: 40, Paths: 12} } + +func liveConfig() Config { + c := DefaultConfig() + c.DryRun = false + return c +} + +func TestDryRunWritesNothing(t *testing.T) { + b, client, _ := setup(t, DefaultConfig(), map[string]string{"use-gzip": "true"}, feed{scanner("203.0.113.5")}) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + cm := current(t, client) + if _, ok := cm.Data["block-cidrs"]; ok || cm.Annotations[AnnotationManaged] != "" { + t.Fatalf("dry run wrote to the configmap: %+v %+v", cm.Data, cm.Annotations) + } + if len(b.last.Candidates) != 1 { + t.Fatalf("dry run should still report candidates: %+v", b.last) + } +} + +func TestBlocksScannerAndPreservesManualEntries(t *testing.T) { + b, client, _ := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.0/24", "use-gzip": "true"}, feed{scanner("203.0.113.5")}) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + cm := current(t, client) + if got := cm.Data["block-cidrs"]; got != "198.51.100.0/24,203.0.113.5" { + t.Fatalf("block-cidrs = %q", got) + } + if cm.Data["use-gzip"] != "true" { + t.Fatal("unrelated keys must be untouched") + } + var state map[string]int64 + if err := json.Unmarshal([]byte(cm.Annotations[AnnotationManaged]), &state); err != nil || len(state) != 1 { + t.Fatalf("annotation = %q (%v)", cm.Annotations[AnnotationManaged], err) + } +} + +func TestNeverBlocksInternalOrAllowlisted(t *testing.T) { + cfg := liveConfig() + cfg.Allow = []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")} + b, client, _ := setup(t, cfg, nil, feed{scanner("127.0.0.1"), scanner("10.244.4.1"), scanner("192.168.1.9"), scanner("203.0.113.77")}) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if v := current(t, client).Data["block-cidrs"]; v != "" { + t.Fatalf("blocked %q", v) + } +} + +func TestThresholdsKeepOutOccasionalVisitors(t *testing.T) { + b, client, _ := setup(t, liveConfig(), nil, feed{ + {Address: "203.0.113.1", Count: 4, Paths: 4}, // too few hits + {Address: "203.0.113.2", Count: 500, Paths: 1}, // one path hammered + {Address: "203.0.113.3", Count: 200, Paths: 60}, // broad scanner + }) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if v := current(t, client).Data["block-cidrs"]; v != "203.0.113.3" { + t.Fatalf("block-cidrs = %q", v) + } +} + +func TestExpiryRemovesOnlyManagedEntries(t *testing.T) { + src := feed{scanner("203.0.113.5")} + b, client, clock := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.7"}, src) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + b.source = feed{} // scanner goes quiet + *clock = clock.Add(25 * time.Hour) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if v := current(t, client).Data["block-cidrs"]; v != "198.51.100.7" { + t.Fatalf("after expiry block-cidrs = %q, want only the manual entry", v) + } +} + +func TestWritesAreRateLimitedAndNewPerTickCapped(t *testing.T) { + cfg := liveConfig() + cfg.MaxNewPerTick = 2 + var many feed + for _, ip := range []string{"203.0.113.1", "203.0.113.2", "203.0.113.3", "203.0.113.4"} { + many = append(many, scanner(ip)) + } + b, client, clock := setup(t, cfg, nil, many) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 { + t.Fatalf("first tick added %d, want 2", n) + } + *clock = clock.Add(time.Minute) // inside MinWrite + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 { + t.Fatalf("write was not rate limited: %d entries", n) + } + *clock = clock.Add(10 * time.Minute) + if err := b.Tick(context.Background()); err != nil { + t.Fatal(err) + } + if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 4 { + t.Fatalf("after the interval got %d entries, want 4", n) + } +} diff --git a/internal/collector/collector_test.go b/internal/collector/collector_test.go index 6cda9ef..4f39d98 100644 --- a/internal/collector/collector_test.go +++ b/internal/collector/collector_test.go @@ -392,6 +392,47 @@ func TestHandleIssuesAndByID(t *testing.T) { } } +func TestIssueTrendsUseTelemetryTimestamps(t *testing.T) { + coll := New(noopBroadcaster{}, 10, nil) + + base := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC) + ld := plog.NewLogs() + rl := ld.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("service.name", "api") + records := rl.ScopeLogs().AppendEmpty().LogRecords() + first := records.AppendEmpty() + first.SetTimestamp(pcommon.NewTimestampFromTime(base)) + first.SetSeverityNumber(plog.SeverityNumberError) + first.Body().SetStr("first failure") + second := records.AppendEmpty() + second.SetTimestamp(pcommon.NewTimestampFromTime(base.Add(time.Hour))) + second.SetSeverityNumber(plog.SeverityNumberError) + second.Body().SetStr("second failure") + + if err := coll.ConsumeLogs(context.Background(), ld); err != nil { + t.Fatalf("ConsumeLogs failed: %v", err) + } + + list := coll.ListIssues(0) + if len(list) != 2 { + t.Fatalf("expected two issues, got %d", len(list)) + } + for _, issue := range list { + if len(issue.Trend) != 1 { + t.Fatalf("expected one trend bucket for %q, got %+v", issue.Title, issue.Trend) + } + } + expected := map[string]string{ + "first failure": issues.HourBucket(base.UnixNano()), + "second failure": issues.HourBucket(base.Add(time.Hour).UnixNano()), + } + for _, issue := range list { + if _, ok := issue.Trend[expected[issue.Title]]; !ok { + t.Fatalf("telemetry timestamp was not used for %q: %+v", issue.Title, issue.Trend) + } + } +} + func TestHandleMLDetectorTest(t *testing.T) { coll := New(nil, 10, nil) req := httptest.NewRequest(http.MethodPost, "/api/ml/test-spike", nil) diff --git a/internal/collector/issue_extract.go b/internal/collector/issue_extract.go index 007bfae..e3df5dc 100644 --- a/internal/collector/issue_extract.go +++ b/internal/collector/issue_extract.go @@ -37,7 +37,14 @@ func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int6 service := spanServiceName(rs.Resource(), span) title := spanErrorTitle(span) found = true - tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title)) + occurrenceTimestamp := span.StartTimestamp() + if occurrenceTimestamp == 0 { + occurrenceTimestamp = span.EndTimestamp() + } + if occurrenceTimestamp == 0 { + occurrenceTimestamp = pcommon.Timestamp(timestamp) + } + tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), int64(occurrenceTimestamp), title)) } } } @@ -102,7 +109,14 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bo title := logErrorTitle(record) found = true kind := "log_error" - sample := logSample(record, rl.Resource(), timestamp, title) + occurrenceTimestamp := record.Timestamp() + if occurrenceTimestamp == 0 { + occurrenceTimestamp = record.ObservedTimestamp() + } + if occurrenceTimestamp == 0 { + occurrenceTimestamp = pcommon.Timestamp(timestamp) + } + sample := logSample(record, rl.Resource(), int64(occurrenceTimestamp), title) if attrString(record.Attributes(), "source") == "frontend" { kind = "frontend_error" sample.URL = attrString(record.Attributes(), "browser.url") diff --git a/internal/honeypot/honeypot.go b/internal/honeypot/honeypot.go index 1e3d4d5..9fc99aa 100644 --- a/internal/honeypot/honeypot.go +++ b/internal/honeypot/honeypot.go @@ -8,11 +8,14 @@ import ( "encoding/json" "net" "net/http" + "net/netip" "os" "strconv" "strings" "sync" "time" + + "git.tornberg.me/go-otel/internal/triage" ) const ( @@ -20,6 +23,7 @@ const ( defaultWindow = 15 * time.Minute defaultTTL = 1 * time.Hour maxTracked = 10000 + maxPathsTracked = 64 ) type Config struct { @@ -28,9 +32,14 @@ type Config struct { TTL time.Duration Delay time.Duration Token string + // MinPaths is how many distinct suspicious paths an IP must request + // before it is promoted. A legitimate client repeatedly fetching one + // odd path (a manifest.json) is not a scanner. Zero means 1. + MinPaths int } type entry struct { + Paths map[string]struct{} Count int FirstSeen time.Time LastSeen time.Time @@ -41,6 +50,7 @@ type entry struct { type IP struct { Address string `json:"ip"` Count int `json:"count"` + Paths int `json:"paths"` FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` PromotedUntil time.Time `json:"promoted_until"` @@ -52,6 +62,7 @@ type Tracker struct { ttl time.Duration delay time.Duration token string + minPaths int mu sync.Mutex clients map[string]entry @@ -73,11 +84,17 @@ func New(cfg Config) *Tracker { if cfg.Delay > 10*time.Second { cfg.Delay = 10 * time.Second } - return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)} + if cfg.MinPaths < 1 { + cfg.MinPaths = 1 + } + return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, minPaths: cfg.MinPaths, clients: make(map[string]entry)} } func ConfigFromEnv() Config { - cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")} + cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN"), MinPaths: 3} + if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_MIN_PATHS")); err == nil && n > 0 { + cfg.MinPaths = n + } if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 { cfg.Threshold = n } @@ -102,7 +119,7 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler { return } ip := clientIP(r) - promoted := t.record(ip) + promoted := t.record(ip, r.URL.Path) if promoted && t.delay > 0 { timer := time.NewTimer(t.delay) select { @@ -125,17 +142,55 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler { }) } -func (t *Tracker) record(ip string) bool { +// Untrackable reports addresses that must never be tracked or blocked: +// loopback, private, link-local, multicast, unspecified or unparsable. Behind +// a proxy hop the observed peer is often 127.0.0.1 or a node address, and +// promoting it would cut off the ingress itself. +func Untrackable(ip string) bool { + addr, err := netip.ParseAddr(strings.TrimSpace(ip)) + if err != nil { + return true + } + addr = addr.Unmap() + if addr.IsLoopback() || addr.IsPrivate() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() || addr.IsMulticast() { + return true + } + for _, p := range reservedPrefixes { + if p.Contains(addr) { + return true + } + } + return false +} + +// reservedPrefixes are ranges the standard library helpers do not cover but +// that are never a remote scanner: carrier-grade NAT (also used by Tailscale +// and mobile carriers), IETF protocol assignments, benchmarking, and the +// reserved/broadcast block. +var reservedPrefixes = []netip.Prefix{ + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("240.0.0.0/4"), +} + +func (t *Tracker) record(ip, path string) bool { + if Untrackable(ip) { + return false + } now := time.Now() t.mu.Lock() defer t.mu.Unlock() state, ok := t.clients[ip] if !ok || now.Sub(state.LastSeen) > t.window { - state = entry{FirstSeen: now} + state = entry{FirstSeen: now, Paths: map[string]struct{}{}} } state.Count++ state.LastSeen = now - if state.Count >= t.threshold { + if len(state.Paths) < maxPathsTracked { + state.Paths[strings.ToLower(path)] = struct{}{} + } + if state.Count >= t.threshold && len(state.Paths) >= t.minPaths { state.PromotedUntil = now.Add(t.ttl) } t.clients[ip] = state @@ -152,10 +207,10 @@ func (t *Tracker) record(ip string) bool { // promoted-IP feed. func (t *Tracker) RecordTelemetry(ip, path string) bool { ip = strings.TrimSpace(strings.SplitN(ip, ",", 2)[0]) - if ip == "" || !isSuspiciousPath(path) { + if ip == "" || !(isSuspiciousPath(path) || triage.IsProbePath(path)) { return false } - return t.record(ip) + return t.record(ip, path) } // PromotedIPs returns a snapshot suitable for an ingress synchronizer. @@ -168,7 +223,7 @@ func (t *Tracker) PromotedIPs() []IP { if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) { continue } - result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil}) + result = append(result, IP{Address: address, Count: state.Count, Paths: len(state.Paths), FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil}) } return result } diff --git a/internal/honeypot/honeypot_test.go b/internal/honeypot/honeypot_test.go index d89c5ce..10b86fc 100644 --- a/internal/honeypot/honeypot_test.go +++ b/internal/honeypot/honeypot_test.go @@ -10,8 +10,8 @@ import ( func TestPromotesAfterThresholdAndExpires(t *testing.T) { tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"}) - for i := 0; i < 3; i++ { - req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil) + for _, path := range []string{"/.env", "/.git/config", "/wp-login.php"} { + req := httptest.NewRequest(http.MethodGet, "https://example.test"+path, nil) req.RemoteAddr = "192.0.2.10:1234" tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req) } @@ -56,3 +56,29 @@ func TestRecordTelemetryPromotesTheObservedSourceIP(t *testing.T) { t.Fatalf("unexpected promoted IPs: %+v", ips) } } + +func TestNeverTracksInternalAddresses(t *testing.T) { + tracker := New(Config{Threshold: 1, TTL: time.Minute}) + for _, ip := range []string{"127.0.0.1", "::1", "10.244.4.1", "192.168.1.5", "169.254.1.1", "100.64.0.1", "100.127.255.254", "198.18.0.1", "240.0.0.1", "255.255.255.255", "::ffff:10.1.2.3", "not-an-ip", ""} { + if tracker.RecordTelemetry(ip, "/.env") { + t.Fatalf("%q must never be promoted", ip) + } + } + if got := len(tracker.PromotedIPs()); got != 0 { + t.Fatalf("promoted %d internal addresses", got) + } +} + +func TestMinPathsRequiresBreadth(t *testing.T) { + tracker := New(Config{Threshold: 2, TTL: time.Minute, MinPaths: 3}) + for i := 0; i < 5; i++ { + tracker.RecordTelemetry("203.0.113.9", "/manifest.json") + } + if len(tracker.PromotedIPs()) != 0 { + t.Fatal("one repeated path must not promote") + } + tracker.RecordTelemetry("203.0.113.9", "/.env") + if !tracker.RecordTelemetry("203.0.113.9", "/wp-login.php") { + t.Fatal("three distinct paths must promote") + } +} diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index 2858798..ffb1827 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -153,6 +153,9 @@ func NewClickHouseStore(ctx context.Context, cfg ClickHouseConfig) (*ClickHouseS // initSchema creates the raw entries table and the materialized hourly // rollup used by the stats endpoints. Idempotent. func (s *ClickHouseStore) initSchema(ctx context.Context) error { + if err := EnsureNormalizedSchema(ctx, s.conn, s.db); err != nil { + return fmt.Errorf("normalized schema: %w", err) + } if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.entries ( timestamp DateTime64(9, 'UTC') NOT NULL, type LowCardinality(String) NOT NULL, @@ -361,6 +364,7 @@ func (s *ClickHouseStore) Append(ctx context.Context, entry websocket.HistoryEnt if err := s.appendSpanFacts(ctx, ts, facts); err != nil { return fmt.Errorf("clickhouse span facts: %w", err) } + s.appendNormalized(ctx, entry) return nil } diff --git a/internal/storage/normalized.go b/internal/storage/normalized.go new file mode 100644 index 0000000..0a94d82 --- /dev/null +++ b/internal/storage/normalized.go @@ -0,0 +1,107 @@ +package storage + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + + "git.tornberg.me/go-otel/internal/normalization" + "git.tornberg.me/go-otel/internal/websocket" +) + +// normalizedColumns is the explicit insert column list. Tables migrated with +// ALTER ADD COLUMN have a different physical column order than a fresh +// CREATE, so positional inserts are not safe. +const normalizedColumns = `event_id, entry_type, event_index, ingest_timestamp, timestamp, service, release, environment, + kind, source, is_error, status_class, status_code, duration_ms, route, span_name, metric_name, metric_type, metric_unit, + metric_value, metric_count, metric_sum, metric_min, metric_max, exception_type, message_normalized, stack_signature, + trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version, + method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode` + +// NormalizedInsertSQL is the INSERT prefix for a normalized_events batch. +func NormalizedInsertSQL(db string) string { + return fmt.Sprintf("INSERT INTO %s.normalized_events (%s)", db, normalizedColumns) +} + +// EnsureNormalizedSchema creates normalized_events and applies the additive +// migrations. Idempotent. +func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) error { + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events ( + event_id String, + entry_type LowCardinality(String), + event_index UInt32, + ingest_timestamp DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String), + kind LowCardinality(String), source LowCardinality(String), is_error UInt8, + status_class LowCardinality(String), status_code UInt16, duration_ms Float64, + route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String, + metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64, + exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String, + is_retry UInt8, is_timeout UInt8, is_cancellation UInt8, + exact_signature String, pattern_signature String, version UInt8 DEFAULT 1, + method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '', + origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT '' + ) ENGINE = ReplacingMergeTree(version) + PARTITION BY toYYYYMM(timestamp) + ORDER BY (service, kind, pattern_signature, timestamp, event_id) + TTL toDateTime(timestamp) + INTERVAL 180 DAY`, db) + if err := conn.Exec(ctx, query); err != nil { + return err + } + // Developer tables predate these columns; historical rows get safe defaults. + for _, col := range []string{ + "issue_fingerprint String DEFAULT ''", "is_retry UInt8 DEFAULT 0", "is_timeout UInt8 DEFAULT 0", + "is_cancellation UInt8 DEFAULT 0", "method String DEFAULT ''", "host String DEFAULT ''", + "user_agent String DEFAULT ''", "origin_host String DEFAULT ''", "referer_host String DEFAULT ''", + "fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''", + } { + if err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS %s", db, col)); err != nil { + return err + } + } + return nil +} + +// AppendNormalizedEvent adds one event to a batch prepared with NormalizedInsertSQL. +func AppendNormalizedEvent(batch driver.Batch, e normalization.Event) error { + return batch.Append( + e.EventID, e.EntryType, e.EventIndex, time.Unix(0, e.IngestTimestamp), time.Unix(0, e.Timestamp), + e.Service, e.Release, e.Environment, e.Kind, e.Source, boolToUint8(e.IsError), + e.StatusClass, uint16(e.StatusCode), e.DurationMS, e.Route, e.SpanName, e.MetricName, e.MetricType, e.MetricUnit, + e.MetricValue, e.MetricCount, e.MetricSum, e.MetricMin, e.MetricMax, + e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.IssueFingerprint, + boolToUint8(e.IsRetry), boolToUint8(e.IsTimeout), boolToUint8(e.IsCancellation), e.ExactSignature, e.PatternSignature, + uint8(1), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode, + ) +} + +// appendNormalized derives and stores normalized rows for one raw entry. +// The raw entry is the source of truth, so a failure here is logged and never +// fails ingest; the backfill tool can rebuild any gap. +func (s *ClickHouseStore) appendNormalized(ctx context.Context, entry websocket.HistoryEntry) { + events, err := normalization.NormalizeEntry(entry.Type, entry.Timestamp, entry.Data) + if err != nil || len(events) == 0 { + return + } + ctx = clickhouse.Context(ctx, clickhouse.WithSettings(clickhouse.Settings{"async_insert": 1, "wait_for_async_insert": 0})) + batch, err := s.conn.PrepareBatch(ctx, NormalizedInsertSQL(s.db)) + if err != nil { + log.Printf("normalized insert prepare: %v", err) + return + } + for _, e := range events { + if err := AppendNormalizedEvent(batch, e); err != nil { + _ = batch.Abort() + log.Printf("normalized insert append: %v", err) + return + } + } + if err := batch.Send(); err != nil { + log.Printf("normalized insert send: %v", err) + } +} diff --git a/internal/triage/triage.go b/internal/triage/triage.go new file mode 100644 index 0000000..9a31f45 --- /dev/null +++ b/internal/triage/triage.go @@ -0,0 +1,107 @@ +// Package triage assigns a cause-level category to a normalized error +// pattern. The categories are deliberately not restatements of status or +// severity: they answer "why did this happen and who should care". +// +// Rules are deterministic and ordered; the first match wins. Structured flags +// (timeout, cancellation, retry) are preferred over message text. +package triage + +import ( + "regexp" + "strings" +) + +type Category string + +const ( + ScannerProbe Category = "scanner_probe" + DependencyUnreachable Category = "dependency_unreachable" + ClientCredentialIssue Category = "client_credential_issue" + StaleReference Category = "stale_reference" + ServiceDefect Category = "service_defect" + CapacitySaturation Category = "capacity_saturation" + ExpectedLifecycle Category = "expected_lifecycle" + NeedsReview Category = "needs_review" +) + +// Input is the non-PII slice of a normalized event the rules look at. +type Input struct { + Service string + Kind string + Status int + Route string + Title string + Message string + IsTimeout bool + IsCancellation bool + IsRetry bool +} + +type Result struct { + Category Category `json:"category"` + Rule string `json:"rule"` // which rule fired, for review and tuning +} + +var ( + // Probe families: secrets/dotfiles, config and backup file types, source + // control, CMS/admin paths, path traversal. Deliberately path-only. + probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` + + `wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` + + `\.php\b|credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json)$|bootstrap\.properties|` + + `\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg)$)`) + bugText = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`) + capText = regexp.MustCompile(`(?i)timeout|deadline exceeded|\bpool\b|memory limit|\boom(killed)?\b|too many|queue full|exhaust`) + depText = regexp.MustCompile(`(?i)dial|connection refused|\beof\b|no such host|upstream|unavailable|broken pipe|reset by peer`) + lifeText = regexp.MustCompile(`(?i)healthz|readyz|/ping|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load`) +) + +// IsProbePath reports whether a request path looks like an automated scan. +// It is the single definition shared by triage and IP-blocking analysis. +func IsProbePath(path string) bool { return probeRoute.MatchString(path) } + +// Classify returns the category for one pattern's example event. +func Classify(in Input) Result { + blob := strings.Join([]string{in.Title, in.Message, in.Route}, " ") + switch { + case probeRoute.MatchString(in.Route): + return Result{ScannerProbe, "probe-route"} + case in.IsCancellation: + return Result{ExpectedLifecycle, "cancellation-flag"} + case in.IsRetry: + return Result{ExpectedLifecycle, "retry-flag"} + case bugText.MatchString(blob): + return Result{ServiceDefect, "bug-text"} + case in.IsTimeout: + return Result{CapacitySaturation, "timeout-flag"} + case capText.MatchString(blob): + return Result{CapacitySaturation, "capacity-text"} + case depText.MatchString(blob): + return Result{DependencyUnreachable, "dependency-text"} + case in.Status == 401: + return Result{ClientCredentialIssue, "status-401"} + case in.Status == 502 || in.Status == 503 || in.Status == 504: + return Result{DependencyUnreachable, "status-5xx-upstream"} + case in.Status >= 500: + return Result{ServiceDefect, "status-5xx"} + case in.Status == 404 || in.Status == 410: + return Result{StaleReference, "status-404"} + case lifeText.MatchString(blob): + return Result{ExpectedLifecycle, "lifecycle-text"} + case in.Status == 499 || in.Status == 200 || in.Status == 202 || in.Status == 204 || (in.Status >= 300 && in.Status < 400): + return Result{ExpectedLifecycle, "benign-status"} + case in.Status == 0 && (in.Kind == "span" || in.Kind == "log"): + return Result{ExpectedLifecycle, "no-status-signal"} + } + return Result{NeedsReview, "no-rule"} +} + +// Actionable reports whether alerts in this category should page a human by +// default. Probes are a security channel rather than an ops one, and the rest +// are expected noise or harmless. +func (c Category) Actionable() bool { + switch c { + case ScannerProbe, StaleReference, ExpectedLifecycle: + return false + } + return true +} diff --git a/internal/triage/triage_test.go b/internal/triage/triage_test.go new file mode 100644 index 0000000..aa9cfda --- /dev/null +++ b/internal/triage/triage_test.go @@ -0,0 +1,43 @@ +package triage + +import "testing" + +func TestClassify(t *testing.T) { + cases := []struct { + name string + in Input + want Category + }{ + {"wp 500 is a probe not an outage", Input{Status: 500, Route: "/wp"}, ScannerProbe}, + {"php 405 is a probe", Input{Status: 405, Route: "/index.php"}, ScannerProbe}, + {"ses.env", Input{Status: 404, Route: "/ses.env"}, ScannerProbe}, + {"yaml config 302", Input{Status: 302, Route: "/config/aws.yml"}, ScannerProbe}, + {"traversal", Input{Status: 404, Route: "/api/w/admins/get_log_file/../../../../proc/self/environ"}, ScannerProbe}, + {"credentials file", Input{Status: 404, Route: "/firebase-credentials.json"}, ScannerProbe}, + {"matrix rooms is not OOM", Input{Status: 200, Route: "/_matrix/client/r0/rooms/x/messages"}, ExpectedLifecycle}, + {"real api 404 stays stale", Input{Status: 404, Route: "/api/config"}, StaleReference}, + {"env 404", Input{Status: 404, Route: "/.env"}, ScannerProbe}, + {"git config 502", Input{Status: 502, Route: "/.git/config"}, ScannerProbe}, + {"poller 401", Input{Status: 401, Route: "/api/repos/available"}, ClientCredentialIssue}, + {"upstream 503", Input{Status: 503, Route: "/store/regions"}, DependencyUnreachable}, + {"real 500", Input{Status: 500, Route: "/"}, ServiceDefect}, + {"panic in message", Input{Message: "panic: nil pointer dereference"}, ServiceDefect}, + {"timeout flag", Input{IsTimeout: true, Message: "query"}, CapacitySaturation}, + {"dial error", Input{Message: "*websocket.dialerror: connection refused"}, DependencyUnreachable}, + {"missing commit", Input{Status: 404, Route: "/mats/repo/commit/{id}"}, StaleReference}, + {"cancellation", Input{IsCancellation: true, Message: "context canceled"}, ExpectedLifecycle}, + {"health", Input{Kind: "span", Status: 200, Route: "/readyz"}, ExpectedLifecycle}, + {"unknown", Input{Kind: "metric", Status: 418}, NeedsReview}, + } + for _, c := range cases { + if got := Classify(c.in); got.Category != c.want { + t.Errorf("%s: got %s (%s), want %s", c.name, got.Category, got.Rule, c.want) + } + } +} + +func TestProbeBeatsLifecycleFlags(t *testing.T) { + if got := Classify(Input{Route: "/.env", IsRetry: true}); got.Category != ScannerProbe { + t.Fatalf("got %s", got.Category) + } +} diff --git a/k8s/clickhouse.yaml b/k8s/clickhouse.yaml index 37f2c91..e077656 100644 --- a/k8s/clickhouse.yaml +++ b/k8s/clickhouse.yaml @@ -143,4 +143,6 @@ spec: - name: http port: 8123 targetPort: 8123 - type: ClusterIP + # LoadBalancer for local-LAN development access only (training exports, + # spike-alerts). The receiver itself stays behind its own Service. + type: LoadBalancer diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 84e8b06..58aaede 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -93,6 +93,16 @@ spec: value: "30s" - name: OTEL_CLICKHOUSE_CONN_MAX_LIFETIME value: "10m" + # Ingress blocker: promotes scanner IPs into ingress-nginx block-cidrs. + # Dry-run first: it logs what it would block and exposes it at + # /api/blocker/status. Set OTEL_BLOCKER_DRY_RUN=false to enforce, and + # put your own public IP(s) in OTEL_BLOCKER_ALLOW before doing so. + - name: OTEL_BLOCKER_ENABLED + value: "true" + - name: OTEL_BLOCKER_DRY_RUN + value: "true" + - name: OTEL_BLOCKER_ALLOW + value: "" - name: OTEL_STORE_WORKERS value: "4" # Optional rollout observation. The watcher is read-only and uses the @@ -229,3 +239,32 @@ subjects: - kind: ServiceAccount name: otel-debug-receiver namespace: monitoring + +--- +# The blocker may read and update exactly one ConfigMap: the ingress-nginx +# controller's, where `block-cidrs` lives. Nothing else in that namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: otel-debug-receiver-ip-blocker + namespace: ingress-nginx +rules: +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["ingress-nginx-controller"] + verbs: ["get", "update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: otel-debug-receiver-ip-blocker + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: otel-debug-receiver-ip-blocker +subjects: +- kind: ServiceAccount + name: otel-debug-receiver + namespace: monitoring diff --git a/otel-ui/src/App.tsx b/otel-ui/src/App.tsx index 5accd18..bb6be85 100644 --- a/otel-ui/src/App.tsx +++ b/otel-ui/src/App.tsx @@ -71,10 +71,10 @@ function App() { typeof window !== "undefined" && (window.location.pathname === "/auth_callback" || window.location.search.includes("code=")); - // The Issues tab is the redesigned triage overview: it owns its own filter - // bar and cards, so the legacy anomaly banner and volume waveform above it - // would just duplicate that UI in a different visual language. - const isTriageOverview = activeTab === "tab0" || activeTab === "tab3" || activeTab === "tab7"; + // The redesigned screens (issues, logs, RUM sessions, rollouts, metrics) own + // their own filter bar and cards, so the legacy anomaly banner and volume + // waveform above them would just duplicate that UI in a different visual language. + const isTriageOverview = activeTab === "tab0" || activeTab === "tab3" || activeTab === "tab7" || activeTab === "tab9" || activeTab === "tab2"; if (isCallback) { return ; diff --git a/otel-ui/src/components/AnomalyRadarBar.tsx b/otel-ui/src/components/AnomalyRadarBar.tsx index 611f3c1..4bb889d 100644 --- a/otel-ui/src/components/AnomalyRadarBar.tsx +++ b/otel-ui/src/components/AnomalyRadarBar.tsx @@ -81,7 +81,7 @@ export const AnomalyRadarBar: React.FC = () => { diff --git a/otel-ui/src/components/GaugeMeter.tsx b/otel-ui/src/components/GaugeMeter.tsx deleted file mode 100644 index 93c92b6..0000000 --- a/otel-ui/src/components/GaugeMeter.tsx +++ /dev/null @@ -1,372 +0,0 @@ -import { useMemo } from "react"; -import { ArrowDownRight, ArrowUpRight, Minus } from "lucide-react"; -import { getServicePalette } from "../lib/otel-utils"; - -export interface GaugeMeterProps { - name: string; - service?: string; - value: number; - min?: number; - max?: number; - avg?: number; - unit?: string; - size?: "sm" | "md" | "lg"; - thresholds?: { - warn: number; - danger: number; - }; - subtitle?: string; - history?: number[]; - showTrend?: boolean; - isSum?: boolean; - aggLabel?: string; - soloPod?: string; - podCount?: number; - podSummaries?: Array<{ pod: string; value: number }>; -} - -const formatNumber = (val: number, maxDecimals = 2) => { - if (Math.abs(val) >= 1_000_000) { - return `${(val / 1_000_000).toFixed(1)}M`; - } - if (Math.abs(val) >= 1_000) { - return `${(val / 1_000).toFixed(1)}k`; - } - if (Number.isInteger(val)) { - return val.toString(); - } - return val.toFixed(maxDecimals); -}; - -// SVG arc calculations -const polarToCartesian = (centerX: number, centerY: number, radius: number, angleInDegrees: number) => { - const angleInRadians = ((angleInDegrees - 90) * Math.PI) / 180.0; - return { - x: centerX + radius * Math.cos(angleInRadians), - y: centerY + radius * Math.sin(angleInRadians), - }; -}; - -const describeArc = (x: number, y: number, radius: number, startAngle: number, endAngle: number) => { - const start = polarToCartesian(x, y, radius, endAngle); - const end = polarToCartesian(x, y, radius, startAngle); - const largeArcFlag = endAngle - startAngle <= 180 ? "0" : "1"; - return ["M", start.x, start.y, "A", radius, radius, 0, largeArcFlag, 0, end.x, end.y].join(" "); -}; - -export const GaugeMeter = ({ - name, - service, - value, - min: customMin, - max: customMax, - avg, - unit = "", - size = "md", - thresholds, - subtitle, - history = [], - showTrend = true, - isSum = false, - aggLabel, - soloPod, - podCount, - podSummaries, -}: GaugeMeterProps) => { - const servicePalette = service ? getServicePalette(service) : null; - // Determine min/max range - const { min, max } = useMemo(() => { - let computedMin = customMin !== undefined ? customMin : 0; - let computedMax = customMax !== undefined ? customMax : 100; - - if (history.length > 0 && customMin === undefined && customMax === undefined) { - const histMin = Math.min(...history); - const histMax = Math.max(...history); - if (histMax <= 1 && histMin >= 0) { - // Percentage ratio 0..1 - computedMin = 0; - computedMax = 1; - } else if (histMax <= 100 && histMin >= 0) { - // 0..100% - computedMin = 0; - computedMax = 100; - } else { - computedMin = Math.floor(Math.min(0, histMin)); - computedMax = Math.ceil(histMax * 1.15) || 100; - } - } - - if (computedMax <= computedMin) { - computedMax = computedMin + 1; - } - - return { min: computedMin, max: computedMax }; - }, [customMin, customMax, history]); - - // Normalized percent 0..100 - const percent = useMemo(() => { - const clamped = Math.max(min, Math.min(value, max)); - return ((clamped - min) / (max - min)) * 100; - }, [value, min, max]); - - // Color determination based on thresholds - const warnThreshold = thresholds?.warn ?? 70; - const dangerThreshold = thresholds?.danger ?? 90; - - const statusColor = useMemo(() => { - if (percent >= dangerThreshold) { - return { - text: "text-ds-critical", - stroke: "var(--ds-critical)", - gradientStart: "var(--ds-critical)", - gradientEnd: "var(--ds-critical)", - bg: "bg-ds-critical-bg", - border: "border-ds-critical/30", - glow: "color-mix(in srgb, var(--ds-critical) 35%, transparent)", - label: "Critical", - }; - } - if (percent >= warnThreshold) { - return { - text: "text-ds-warn", - stroke: "var(--ds-warn)", - gradientStart: "var(--ds-warn)", - gradientEnd: "var(--ds-warn)", - bg: "bg-ds-warn-bg", - border: "border-ds-warn/30", - glow: "color-mix(in srgb, var(--ds-warn) 35%, transparent)", - label: "Warning", - }; - } - return { - text: "text-ds-accent", - stroke: "var(--ds-accent)", - gradientStart: "var(--ds-accent)", - gradientEnd: "var(--ds-accent)", - bg: "bg-ds-accent-bg", - border: "border-ds-accent/30", - glow: "color-mix(in srgb, var(--ds-accent) 35%, transparent)", - label: "Nominal", - }; - }, [percent, warnThreshold, dangerThreshold]); - - // Trend calculation - const trend = useMemo(() => { - if (!showTrend || history.length < 2) return null; - const prev = history[history.length - 2]; - const diff = value - prev; - if (Math.abs(diff) < 0.0001) return { direction: "flat", text: "steady", color: "text-ds-text-3" }; - if (diff > 0) return { direction: "up", text: `+${formatNumber(diff)}`, color: "text-ds-critical" }; - return { direction: "down", text: `${formatNumber(diff)}`, color: "text-ds-ok" }; - }, [value, history, showTrend]); - - // Dimensions - const dim = size === "sm" ? 120 : size === "lg" ? 220 : 160; - const strokeWidth = size === "sm" ? 8 : size === "lg" ? 14 : 10; - const radius = (dim - strokeWidth * 2) / 2; - const centerX = dim / 2; - const centerY = dim / 2 + (size === "sm" ? 4 : 8); - - // 240 degree arc from -120 to +120 degrees - const startAngle = -120; - const endAngle = 120; - const totalAngle = endAngle - startAngle; - const valueAngle = startAngle + (percent / 100) * totalAngle; - - const bgArc = describeArc(centerX, centerY, radius, startAngle, endAngle); - const valueArc = describeArc(centerX, centerY, radius, startAngle, Math.min(valueAngle, endAngle)); - - // Ticks at 0, 50, 100 - const tickAngles = [startAngle, startAngle + totalAngle * 0.5, endAngle]; - - const gaugeId = useMemo( - () => `${service ? service.replace(/[^a-zA-Z0-9]/g, "_") + "_" : ""}${name.replace(/[^a-zA-Z0-9]/g, "_")}`, - [service, name] - ); - - return ( -
-
-
-
- {service && ( - - - {service} - - )} - {podCount !== undefined && podCount > 1 && ( - - ∑ - {podCount} pods - - )} -
- - {name} - -
- - {statusColor.label} - -
- -
- - - - - - - - - - - - {/* Background Track */} - - - {/* Ticks */} - {tickAngles.map((angle, idx) => { - const inner = polarToCartesian(centerX, centerY, radius - strokeWidth * 0.8, angle); - const outer = polarToCartesian(centerX, centerY, radius + strokeWidth * 0.8, angle); - return ( - - ); - })} - - {/* Active Value Arc */} - {percent > 0 && ( - - )} - - {/* Needle / Value dot indicator */} - {percent > 0 && (() => { - const tip = polarToCartesian(centerX, centerY, radius, valueAngle); - return ( - - ); - })()} - - - {/* Center Readout */} -
- {soloPod ? ( - - {soloPod} - - ) : aggLabel ? ( - - {aggLabel} - - ) : podCount !== undefined && podCount > 1 ? ( - - {isSum ? "Sum (∑)" : "Avg (μ)"} - - ) : null} -
- - {formatNumber(value)} - - {unit && ( - - {unit} - - )} -
- {podCount !== undefined && podCount > 1 ? ( - - {isSum ? `~${formatNumber(value / podCount)}/pod` : `${podCount} pods avg`} - - ) : subtitle ? ( - - {subtitle} - - ) : null} -
-
- - {/* Footer range indicators & trend */} -
- - min {formatNumber(min)} - - {trend && ( - - {trend.direction === "up" && } - {trend.direction === "down" && } - {trend.direction === "flat" && } - {trend.text} - - )} - {avg !== undefined && ( - - avg {formatNumber(avg)} - - )} - - max {formatNumber(max)} - -
- - {/* Pod Breakdown pills if available */} - {podSummaries && podSummaries.length > 1 && ( -
- {podSummaries.slice(0, 3).map((ps) => ( -
- {ps.pod} - {formatNumber(ps.value)} -
- ))} - {podSummaries.length > 3 && ( - - +{podSummaries.length - 3} - - )} -
- )} -
- ); -}; diff --git a/otel-ui/src/components/MetricsVisualizer.tsx b/otel-ui/src/components/MetricsVisualizer.tsx index f9e7873..634a0c6 100644 --- a/otel-ui/src/components/MetricsVisualizer.tsx +++ b/otel-ui/src/components/MetricsVisualizer.tsx @@ -1,930 +1,832 @@ import { useMemo, useState } from "react"; import { - AreaChart, Area, + AreaChart, + CartesianGrid, Line, + LineChart, + ResponsiveContainer, + Tooltip, XAxis, YAxis, - CartesianGrid, - Tooltip, - ResponsiveContainer, } from "recharts"; -import { Activity, BarChart3, Sparkles, Filter, Gauge, Search, X, Boxes, Eye } from "lucide-react"; -import { useMetricsDetails } from "../hooks/useMetrics"; -import type { MetricDataPoint, OtelAttribute } from "../lib/types"; -import { formatAttributeValue, getServicePalette, getPodPalette } from "../lib/otel-utils"; -import { GaugeMeter } from "./GaugeMeter"; +import { Activity, BarChart3, Gauge, ListFilter } from "lucide-react"; +import { useMetricsDetails, type MetricDetails } from "../hooks/useMetrics"; +import { useObservabilityData } from "../hooks/useObservabilityData"; +import type { MetricDataPoint } from "../lib/types"; +import { getPodPalette, POD_COLORS } from "../lib/otel-utils"; +import { FilterBar, type FilterChip } from "./FilterBar"; +import { Segmented } from "./common/Segmented"; +import { ResizablePanes } from "./panes/ResizablePanes"; -const valueFormatter = new Intl.NumberFormat("en-US", { - maximumFractionDigits: 3, -}); +type TypeFilter = "all" | "gauge" | "sum" | "histogram"; +type AggMode = "avg" | "pods" | "sum" | "max"; +const valueFormatter = new Intl.NumberFormat("en-US", { maximumFractionDigits: 3 }); const compactFormatter = new Intl.NumberFormat("en-US", { maximumFractionDigits: 2, notation: "compact", }); -const resolveValue = (point: MetricDataPoint) => { - if (typeof point.asDouble === "number") return point.asDouble; - if (point.asInt !== undefined) return Number(BigInt(point.asInt)); +const fmt = (v: number) => valueFormatter.format(v); +const fmtCompact = (v: number) => compactFormatter.format(v); + +/** OTLP units carry UCUM annotations ("{calls}") and a dimensionless "1" — clean those for display. */ +const displayUnit = (unit?: string): string | undefined => { + if (!unit) return undefined; + const cleaned = unit.replace(/[{}]/g, "").trim(); + return cleaned === "" || cleaned === "1" ? undefined : cleaned; +}; + +const pointValue = (p: MetricDataPoint): number => { + if (typeof p.asDouble === "number") return p.asDouble; + if (p.asInt !== undefined) return Number(BigInt(p.asInt)); return 0; }; -interface MetricCardProps { - name: string; - service: string; - type: "gauge" | "sum" | "histogram"; - points: MetricDataPoint[]; - unit?: string; - pods?: string[]; - index: number; +const pointTime = (p: MetricDataPoint): number => + Number(BigInt(p.timeUnixNano ?? "0") / 1_000_000n); + +const timeLabel = (t: number) => + new Date(t).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); + +/** Deterministic muted hues for multi-series charts (documented data-viz exception). */ +const QUANTILE_SERIES = [ + { key: "p50", label: "p50", stroke: POD_COLORS[0].stroke }, + { key: "p95", label: "p95", stroke: POD_COLORS[3]?.stroke ?? POD_COLORS[0].stroke }, + { key: "max", label: "max", stroke: POD_COLORS[5]?.stroke ?? POD_COLORS[0].stroke }, +] as const; + +/** + * Quantile estimated from one histogram data point's cumulative buckets by + * linear interpolation inside the matching bucket. Returns null when the + * point carries no usable bucket data. + */ +function estimateQuantile(p: MetricDataPoint, q: number): number | null { + const counts = (p.bucketCounts ?? []).map(Number); + const bounds = p.explicitBounds ?? []; + if (counts.length < 2 || bounds.length === 0) return null; + const total = counts[counts.length - 1]; + if (!(total > 0)) return null; + const target = q * total; + let prev = 0; + for (let i = 0; i < counts.length; i++) { + if (counts[i] >= target) { + const lower = i === 0 ? (bounds[0] ?? 0) : (bounds[i - 1] ?? 0); + const upper = i < bounds.length ? bounds[i] : (p.max ?? bounds[i - 1] ?? lower); + const frac = (target - prev) / Math.max(counts[i] - prev, 1); + return lower + (upper - lower) * frac; + } + prev = counts[i]; + } + return null; } -const MetricCard = ({ name, service, type, points, unit, pods = [], index }: MetricCardProps) => { - const [isExpanded, setIsExpanded] = useState(false); - const [viewMode, setViewMode] = useState<"chart" | "gauge">("chart"); - const [selectedPod, setSelectedPod] = useState("all"); +interface ValueRow { + time: number; + value: number; + min: number; + max: number; + avg: number; + sum: number; + [pod: string]: number; +} - const isGaugeLike = useMemo(() => { - if (type === "gauge") return true; - const lower = name.toLowerCase(); - return ( - lower.includes("cpu") || - lower.includes("memory") || - lower.includes("usage") || - lower.includes("ratio") || - lower.includes("utilization") || - lower.includes("duration") || - lower.includes("latency") || - lower.includes("rate") || - lower.includes("load") || - lower.includes("temp") || - unit === "%" - ); - }, [type, name, unit]); +/** + * Time-aligned series for gauge/sum metrics. With several pods, points are + * bucketed with last-known-value hold so per-pod scrape jitter doesn't sawtooth. + */ +function buildValueSeries( + points: MetricDataPoint[], + pods: string[], + agg: AggMode, +): ValueRow[] { + const sorted = [...points].sort((a, b) => pointTime(a) - pointTime(b)); + if (pods.length <= 1) { + return sorted.map((p) => { + const v = pointValue(p); + return { time: pointTime(p), value: v, min: v, max: v, avg: v, sum: v }; + }); + } - const [aggMode, setAggMode] = useState<"avg" | "lines" | "sum" | "max" | "stacked">( - isGaugeLike ? "avg" : "sum" + const times = sorted.map(pointTime).filter((t) => t > 0); + if (times.length === 0) return []; + const minT = times[0]; + const span = Math.max(times[times.length - 1] - minT, 1000); + const buckets = Math.min(60, Math.max(12, Math.floor(sorted.length / pods.length))); + const bucketMs = span / buckets; + + const perBucket: Array> = Array.from({ length: buckets }, () => ({})); + sorted.forEach((p) => { + const t = pointTime(p); + const idx = Math.min(buckets - 1, Math.max(0, Math.floor((t - minT) / bucketMs))); + perBucket[idx][p.pod || pods[0]] = pointValue(p); + }); + + const last: Record = {}; + const rows: ValueRow[] = []; + for (let i = 0; i < buckets; i++) { + pods.forEach((pod) => { + const v = perBucket[i][pod]; + if (v !== undefined) last[pod] = v; + }); + const vals = pods.map((pod) => last[pod]).filter((v): v is number => v !== undefined); + if (vals.length === 0) continue; + const sum = vals.reduce((a, b) => a + b, 0); + const row: ValueRow = { + time: Math.round(minT + (i + 0.5) * bucketMs), + value: agg === "sum" ? sum : agg === "max" ? Math.max(...vals) : sum / vals.length, + sum, + avg: sum / vals.length, + min: Math.min(...vals), + max: Math.max(...vals), + }; + pods.forEach((pod) => { + if (last[pod] !== undefined) row[pod] = last[pod]; + }); + rows.push(row); + } + return rows; +} + +interface HistRow { + time: number; + p50: number | null; + p95: number | null; + max: number | null; + mean: number | null; +} + +/** Window mean of one histogram point (sum ÷ count) — the fallback when a point has no buckets. */ +const histPointMean = (p: MetricDataPoint): number | null => { + const count = Number(p.count ?? 0); + return count > 0 && typeof p.sum === "number" ? p.sum / count : null; +}; + +interface HistogramSeries { + /** "quantiles" when bucket data allowed p50/p95 estimates, else the mean fallback. */ + mode: "quantiles" | "mean"; + rows: HistRow[]; +} + +const meanOf = (vals: Array): number | null => { + const v = vals.filter((x): x is number => x !== null); + return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; +}; + +/** Groups time-ordered items into ≤60 buckets; returns [bucket center time, items]. */ +function bucketize(items: T[]): Array<[number, T[]]> { + if (items.length === 0) return []; + const minT = items[0].time; + const span = Math.max(items[items.length - 1].time - minT, 1000); + const buckets = Math.min(60, Math.max(12, items.length)); + const bucketMs = span / buckets; + const groups: Array<[number, T[]]> = []; + for (let i = 0; i < buckets; i++) { + const lo = minT + i * bucketMs; + const hi = lo + bucketMs; + const inBucket = items.filter((e) => e.time >= lo && (i === buckets - 1 ? e.time <= hi : e.time < hi)); + if (inBucket.length > 0) groups.push([Math.round(lo + bucketMs / 2), inBucket]); + } + return groups; +} + +function buildHistogramSeries(points: MetricDataPoint[]): HistogramSeries { + const ests: HistRow[] = [...points] + .sort((a, b) => pointTime(a) - pointTime(b)) + .map((p) => ({ + time: pointTime(p), + p50: estimateQuantile(p, 0.5), + p95: estimateQuantile(p, 0.95), + max: p.max ?? null, + mean: histPointMean(p), + })) + .filter((e) => e.time > 0); + + // Points stripped of bucket data (some collector paths) can't yield + // quantiles; if none can, chart the per-window mean — the only honest + // series left — instead of three dead lines. + if (!ests.some((e) => e.p50 !== null || e.p95 !== null)) { + return { + mode: "mean", + rows: bucketize(ests).map(([time, group]) => ({ + time, + p50: null, + p95: null, + max: null, + mean: meanOf(group.map((e) => e.mean)), + })), + }; + } + + return { + mode: "quantiles", + rows: bucketize(ests).map(([time, group]) => ({ + time, + p50: meanOf(group.map((e) => e.p50)), + p95: meanOf(group.map((e) => e.p95)), + max: meanOf(group.map((e) => e.max)), + mean: null, + })), + }; +} + +/** Downsample a value list to ~`target` points for a tiny sparkline. */ +function downsample(values: number[], target = 40): number[] { + if (values.length <= target) return values; + const out: number[] = []; + const stride = (values.length - 1) / (target - 1); + for (let i = 0; i < target; i++) out.push(values[Math.round(i * stride)]); + return out; +} + +const Sparkline = ({ values, active }: { values: number[]; active: boolean }) => { + const w = 64; + const h = 20; + if (values.length < 2) return