diff --git a/cmd/spike-alerts/main.go b/cmd/spike-alerts/main.go index bf6befd..742d483 100644 --- a/cmd/spike-alerts/main.go +++ b/cmd/spike-alerts/main.go @@ -65,6 +65,14 @@ func main() { log.Fatal(err) } cfg.DataSince = alerts.DataSince(ctx, conn, *database) + features.MarkBadActors(events, features.BadActorConfig{}) + badActors := map[string]int{} + for _, e := range events { + if e.BadActor != "" { + badActors[e.BadActor]++ + } + } + log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events)) log.Printf("loaded %d error events", len(events)) cache := map[string]classification{} @@ -73,7 +81,11 @@ func main() { muted := map[triage.Category]int{} fired := 0 for at := first.Add(*window); !at.After(end); at = at.Add(*step) { - for _, spike := range suppress.Filter(alerts.FilterActionable(features.DetectSpikes(events, at, cfg), *showMuted, muted)) { + detected := alerts.DetectAlerts(events, at, cfg) // exactly what the server alerts on + if *showMuted { + detected = alerts.FilterActionable(features.DetectSpikes(events, at, cfg), true, muted) + } + for _, spike := range suppress.Filter(detected) { fired++ var cls []classification for _, p := range spike.Patterns { diff --git a/cmd/window-features/main.go b/cmd/window-features/main.go index 347233e..ae4224b 100644 --- a/cmd/window-features/main.go +++ b/cmd/window-features/main.go @@ -107,5 +107,6 @@ func readEvents(path string) []features.Event { if err := scanner.Err(); err != nil { log.Fatal(err) } + features.MarkBadActors(result, features.BadActorConfig{}) return result } diff --git a/docs/STATUS.md b/docs/STATUS.md index 3c4494a..320a057 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -143,6 +143,22 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` orchestrator pattern signature); `HasUpstream` is loaded but not yet a triage rule; trace propagation is missing in cart/checkout/profile/scheduling (roots only). +- Alerts (`internal/alerts`, env `OTEL_ALERT_*`, secret `otel-alerts`): the server + runs the spike detector every minute and posts to Telegram (and/or a generic + webhook). First tick after a start only primes the suppressor. `GET /api/alerts`, + `POST /api/alerts/test`, `go run ./cmd/alert-test`. Recent alerts are in memory only. +- Bad actors (`features.MarkBadActors`): an event is marked, never dropped, when its + client hit >=3 probe paths (`probe-scanner`), walked >=15 not-found routes + (`enumerator`), or is a declared crawler/agent/scanner getting a 4xx (`bot-4xx`; + their 5xx stay visible). Marked events are ignored by `DetectSpikes`/alerts and + counted in `Row.BadActorErrors`. **`Row.Label` now targets the clean future error + rate** (logical requests, no bad actors); `RawLabel` is the old raw-count label. AI + crawler rows stay in the data on purpose (see memory: flag, not block). + Client rules need a resolved client IP, so they only cover data since the trace + columns shipped; the bot rule works on any row with a UA class. +- Route ids: `term_` style segments normalize to `{id}` (per-session websocket + routes were creating a new pattern each session). + ## Commands ```bash diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 009ca1d..e588ae3 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -128,3 +128,38 @@ func TestTelegramTruncates(t *testing.T) { t.Fatalf("message too long: %d", n) } } + +func TestScannerClientIsMutedEntirely(t *testing.T) { + at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) + cur := at.Add(-2 * time.Minute) + scanner := func(route string, n int) []features.Event { + ev := burst("p-"+route, n, cur, 503, route) + for i := range ev { + ev[i].ClientIP = "198.51.100.9" + } + return ev + } + var events []features.Event + for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} { + events = append(events, scanner(r, 4)...) + } + events = append(events, scanner("/", 12)...) // the scanner's innocent-looking root request + if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 { + t.Fatalf("scanner traffic alerted: %+v", got) + } + // a different client with the same 503 on "/" still alerts + other := burst("real-503", 12, cur, 503, "/") + if got := DetectAlerts(other, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 1 { + t.Fatalf("real failure was muted: %+v", got) + } +} + +func TestMinExcessAppliesAfterTriage(t *testing.T) { + at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) + cur := at.Add(-2 * time.Minute) + // 200 probe errors + 6 real ones: total excess is large, actionable excess is 6 < 10 + events := append(burst("probe", 200, cur, 404, "/.env"), burst("real", 6, cur, 503, "/api")...) + if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}); len(got) != 0 { + t.Fatalf("paged on excess that was mostly probes: %+v", got) + } +} diff --git a/internal/alerts/runner.go b/internal/alerts/runner.go index e8180db..84dd58b 100644 --- a/internal/alerts/runner.go +++ b/internal/alerts/runner.go @@ -198,7 +198,21 @@ func (r *Runner) Tick(ctx context.Context, now time.Time) { // DetectAlerts is the detection half of a tick: spikes reduced to actionable // patterns. Exported so the CLI and the runner cannot drift apart. func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfig) []features.Spike { - return FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) + // Scanners, route enumerators and crawlers getting 4xx are noise for paging. + features.MarkBadActors(events, features.BadActorConfig{}) + minExcess := cfg.MinExcess + if minExcess <= 0 { + minExcess = features.DefaultSpikeConfig().MinExcess + } + var out []features.Spike + // The excess threshold is applied after triage: a service whose total + // excess is mostly probes must not page on the little that is left. + for _, sp := range FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) { + if sp.Excess >= float64(minExcess) { + out = append(out, sp) + } + } + return out } func (r *Runner) deliver(ctx context.Context, a Alert) { diff --git a/internal/features/badactor.go b/internal/features/badactor.go new file mode 100644 index 0000000..86c6e04 --- /dev/null +++ b/internal/features/badactor.go @@ -0,0 +1,71 @@ +package features + +import ( + "git.tornberg.me/go-otel/internal/triage" + "git.tornberg.me/go-otel/internal/uaclass" +) + +// Reasons an event is attributed to a bad actor. The empty string means it is +// not. These are noise for alerting and for the model's label, not verdicts: +// the rows are kept, so crawler traffic stays available as training data. +const ( + BadActorProbe = "probe-scanner" // client hit several probe paths + BadActorEnumerator = "enumerator" // client walked many routes that do not exist + BadActorBot = "bot-4xx" // self-declared crawler/agent/scanner getting a client error +) + +// BadActorConfig sets the per-client thresholds, evaluated over the events +// passed to MarkBadActors (a detector range or an export). +type BadActorConfig struct { + ProbePaths int // distinct probe paths that make a client a scanner + EnumPaths int // distinct not-found routes that make a client an enumerator +} + +func DefaultBadActorConfig() BadActorConfig { return BadActorConfig{ProbePaths: 3, EnumPaths: 15} } + +// MarkBadActors sets Event.BadActor in place. +// +// Client rules need a resolved ClientIP (see internal/tracectx) and apply to +// every event of the client, so a scanner's innocent-looking 503 on "/" is +// covered too. The bot rule needs only the User-Agent class, and covers 4xx +// only: a crawler that provokes a 5xx has found a real defect. +func MarkBadActors(events []Event, cfg BadActorConfig) { + d := DefaultBadActorConfig() + if cfg.ProbePaths <= 0 { + cfg.ProbePaths = d.ProbePaths + } + if cfg.EnumPaths <= 0 { + cfg.EnumPaths = d.EnumPaths + } + probe := map[string]map[string]struct{}{} + missing := map[string]map[string]struct{}{} + add := func(m map[string]map[string]struct{}, ip, route string) { + if m[ip] == nil { + m[ip] = map[string]struct{}{} + } + m[ip][route] = struct{}{} + } + for _, e := range events { + if e.ClientIP == "" || e.IsDuplicate { + continue + } + if triage.IsProbePath(e.Route) { + add(probe, e.ClientIP, e.Route) + } + switch e.StatusCode { + case 400, 404, 405, 410: + add(missing, e.ClientIP, e.Route) + } + } + for i := range events { + e := &events[i] + switch { + case e.ClientIP != "" && len(probe[e.ClientIP]) >= cfg.ProbePaths: + e.BadActor = BadActorProbe + case e.ClientIP != "" && len(missing[e.ClientIP]) >= cfg.EnumPaths: + e.BadActor = BadActorEnumerator + case uaclass.IsCrawler(e.UAClass) && e.StatusCode >= 400 && e.StatusCode < 500: + e.BadActor = BadActorBot + } + } +} diff --git a/internal/features/features.go b/internal/features/features.go index f14ee08..06d10ad 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -41,6 +41,7 @@ type Event struct { IsDuplicate bool // repeats an ancestor span's failure; not a new request IsRootCause bool // error span with no erroring child HasUpstream bool // an ingress span that had a child in another service + BadActor string // reason this event is attributed to a bad actor (see MarkBadActors); empty if not } // Config controls feature and label windows. A zero Config receives the @@ -116,13 +117,24 @@ type Row struct { TopClientShare float64 `json:"topClientShare"` BotErrorShare float64 `json:"botErrorShare"` + // Clean errors are logical requests (no duplicate hops) not attributed to a + // bad actor. Label targets the clean future error rate, so scanner and + // crawler noise does not teach the model to alert; RawLabel is the old + // definition on raw error counts, kept for comparison. + BadActorErrors int `json:"badActorErrors"` + CleanErrors int `json:"cleanErrors"` + BaselineErrorRate float64 `json:"baselineErrorRate"` + BaselineCleanRate float64 `json:"baselineCleanRate"` BaselineSamples int `json:"baselineSamples"` FutureEventCount int `json:"futureEventCount"` FutureErrorCount int `json:"futureErrorCount"` FutureErrorRate float64 `json:"futureErrorRate"` + FutureCleanErrors int `json:"futureCleanErrors"` + FutureCleanRate float64 `json:"futureCleanRate"` LabelKnown bool `json:"labelKnown"` Label bool `json:"label"` + RawLabel bool `json:"rawLabel"` } // Build aggregates every service/release over rolling observation windows. @@ -181,16 +193,18 @@ func Build(input []Event, cfg Config) []Row { row.NewIssueCount++ } } - futureTotal, futureErrors := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service) - baselineRate, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service) - row.FutureEventCount, row.FutureErrorCount = futureTotal, futureErrors + futureTotal, futureErrors, futureClean := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service) + baselineRate, baselineClean, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service) + row.FutureEventCount, row.FutureErrorCount, row.FutureCleanErrors = futureTotal, futureErrors, futureClean row.FutureErrorRate = rate(futureErrors, futureTotal) - row.BaselineErrorRate, row.BaselineSamples = baselineRate, samples + row.FutureCleanRate = rate(futureClean, futureTotal) + row.BaselineErrorRate, row.BaselineCleanRate, row.BaselineSamples = baselineRate, baselineClean, samples // A target is only known once the input has reached its exclusive // end. This deliberately withholds the tail of an export instead of // treating an incomplete future interval as a negative label. row.LabelKnown = samples >= cfg.MinBaselineSamples && !start.Add(cfg.Window).Add(cfg.Future).After(observedUntil) - row.Label = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate + row.RawLabel = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate + row.Label = row.LabelKnown && futureClean >= cfg.MinFutureErrors && row.FutureCleanRate >= cfg.SpikeMultiple*baselineClean rows = append(rows, row) } } @@ -250,7 +264,12 @@ func (a *accumulator) add(e Event) { a.value.ErrorCount++ if e.IsDuplicate { a.value.DuplicateErrors++ - } else if e.ClientIP != "" { + } else if e.BadActor != "" { + a.value.BadActorErrors++ + } else { + a.value.CleanErrors++ + } + if !e.IsDuplicate && e.ClientIP != "" { a.clients[e.ClientIP]++ } if e.IsRootCause { @@ -310,35 +329,41 @@ func (a *accumulator) row() Row { return a.value } -func count(events []Event, start, end time.Time, service string) (total, errors int) { +// count returns events, raw errors and clean errors (logical requests not +// attributed to a bad actor) for a service in [start, end). +func count(events []Event, start, end time.Time, service string) (total, errors, clean int) { for _, e := range events { if e.Service == service && !e.Timestamp.Before(start) && e.Timestamp.Before(end) { total++ if e.IsError { errors++ + if !e.IsDuplicate && e.BadActor == "" { + clean++ + } } } } - return total, errors + return total, errors, clean } // baseline samples the same UTC time window on the preceding calendar days. // All intervals end before the current observation start, so no future data // leaks into a training label. -func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (float64, int) { - var total, errors, samples int +func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (raw, clean float64, samples int) { + var total, errors, cleanErrors int for day := 1; day <= days; day++ { from := start.AddDate(0, 0, -day) to := from.Add(window) - countTotal, countErrors := count(events, from, to, service) + countTotal, countErrors, countClean := count(events, from, to, service) if countTotal == 0 { continue } total += countTotal errors += countErrors + cleanErrors += countClean samples++ } - return rate(errors, total), samples + return rate(errors, total), rate(cleanErrors, total), samples } func percentile(values []float64, p float64) float64 { diff --git a/internal/features/features_test.go b/internal/features/features_test.go index d0a4b12..3219092 100644 --- a/internal/features/features_test.go +++ b/internal/features/features_test.go @@ -75,3 +75,35 @@ func TestBuildLogicalRequestFeatures(t *testing.T) { t.Fatalf("bot share wrong: %v", r.BotErrorShare) } } + +func TestBuildLabelIgnoresBadActorNoise(t *testing.T) { + base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC) + var events []Event + mk := func(at time.Time, err bool, bad string) Event { + return Event{Timestamp: at, Service: "api", Kind: "span", IsError: err, BadActor: bad, PatternSignature: "p"} + } + // seven days of a quiet baseline, then a future window of pure bot errors + for d := 0; d <= 7; d++ { + day := base.AddDate(0, 0, d) + for i := 0; i < 50; i++ { + events = append(events, mk(day.Add(time.Duration(i)*time.Second), i == 0, "")) + } + } + target := base.AddDate(0, 0, 7) + for i := 0; i < 30; i++ { + events = append(events, mk(target.Add(6*time.Minute+time.Duration(i)*time.Second), true, BadActorBot)) + } + rows := Build(events, Config{ObservedUntil: target.Add(time.Hour)}) + var found bool + for _, r := range rows { + if r.WindowStart.Equal(target) && r.Service == "api" { + found = true + if !r.LabelKnown || !r.RawLabel || r.Label { + t.Fatalf("raw label should fire on bot errors and clean label must not: %+v", r) + } + } + } + if !found { + t.Fatal("target row missing") + } +} diff --git a/internal/features/spike.go b/internal/features/spike.go index 5bebc51..e8f5944 100644 --- a/internal/features/spike.go +++ b/internal/features/spike.go @@ -94,10 +94,11 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike { } services := map[string]*svc{} for _, e := range events { + // Bad actors (scanners, crawlers getting 4xx) are noise for alerting. // A duplicate is a later hop of a request whose outermost error is // already counted; counting it again multiplies every failure by the // number of services it crossed. - if !e.IsError || e.IsDuplicate { + if !e.IsError || e.IsDuplicate || e.BadActor != "" { continue } ts := e.Timestamp.UTC() diff --git a/internal/features/spike_test.go b/internal/features/spike_test.go index d8f2ad1..1c016b3 100644 --- a/internal/features/spike_test.go +++ b/internal/features/spike_test.go @@ -121,3 +121,37 @@ func TestDetectSpikesSkipsDuplicateHopsAndCountsClients(t *testing.T) { t.Fatalf("clients = %d, want 2", got[0].Patterns[0].Clients) } } + +func TestMarkBadActors(t *testing.T) { + at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) + ev := func(ip, route string, status int, class string) Event { + return Event{Timestamp: at, Service: "nginx", IsError: true, ClientIP: ip, Route: route, StatusCode: status, UAClass: class} + } + var events []Event + for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} { // probe scanner + events = append(events, ev("198.51.100.1", r, 404, "browser")) + } + events = append(events, ev("198.51.100.1", "/", 503, "browser")) // its innocent request + for i := 0; i < 15; i++ { // enumerator of ordinary-looking paths + events = append(events, ev("198.51.100.2", "/api/x"+string(rune('a'+i)), 404, "browser")) + } + events = append(events, + ev("203.0.113.5", "/missing", 404, "ai_crawler"), // declared crawler 4xx: bad actor + ev("203.0.113.5", "/api/orders", 500, "ai_crawler"), // ...but its 5xx is a real defect + ev("203.0.113.6", "/api/orders", 500, "browser"), + ev("203.0.113.7", "/missing", 404, "library"), // libraries are not crawlers + ) + MarkBadActors(events, BadActorConfig{}) + got := map[string]int{} + for _, e := range events { + got[e.BadActor]++ + } + if got[BadActorProbe] != 4 || got[BadActorEnumerator] != 15 || got[BadActorBot] != 1 || got[""] != 3 { + t.Fatalf("marking wrong: %+v", got) + } + for _, e := range events { + if e.ClientIP == "203.0.113.5" && e.StatusCode == 500 && e.BadActor != "" { + t.Fatal("a crawler-provoked 5xx must stay visible") + } + } +} diff --git a/internal/issues/route_test.go b/internal/issues/route_test.go new file mode 100644 index 0000000..2dae57d --- /dev/null +++ b/internal/issues/route_test.go @@ -0,0 +1,18 @@ +package issues + +import "testing" + +func TestNormalizeRoutePrefixedIDs(t *testing.T) { + cases := map[string]string{ + "/api/terminals/term_ab315d5ce24c4662b34a8ad998be79cf/ws": "/api/terminals/{id}/ws", + "/api/sessions/sess-9f2c41aa77/events": "/api/sessions/{id}/events", + "/api/skills": "/api/skills", + "/api/v2/settings": "/api/v2/settings", + "/api/user_settings": "/api/user_settings", + } + for in, want := range cases { + if got := NormalizeRoute(in); got != want { + t.Errorf("NormalizeRoute(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/issues/tracker.go b/internal/issues/tracker.go index bc6730a..b5ac7b9 100644 --- a/internal/issues/tracker.go +++ b/internal/issues/tracker.go @@ -346,7 +346,7 @@ func NormalizeRoute(route string) string { if part == "" { continue } - if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) { + if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) || prefixedIDRe.MatchString(part) { parts[i] = "{id}" } } @@ -623,6 +623,9 @@ var ( hexIDRe = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`) httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`) numRe = regexp.MustCompile(`\b\d+\b`) + // A route segment that is a short prefix plus a long hex id (term_ab31…, + // sess-9f2c…): \b does not fire between "_" and a hex digit, so hexIDRe misses it. + prefixedIDRe = regexp.MustCompile(`(?i)^[a-z]{1,12}[_-][0-9a-f]{8,}$`) spaceRe = regexp.MustCompile(`\s+`) ) diff --git a/internal/triage/triage.go b/internal/triage/triage.go index 57772fd..53a2b59 100644 --- a/internal/triage/triage.go +++ b/internal/triage/triage.go @@ -47,7 +47,8 @@ var ( // control, CMS/admin paths, path traversal. Deliberately path-only. probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` + `wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` + - `\.php\b|_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` + + `\.php\b|\.codex|\.cursor|\.windsurf|\.aider|\.clinerules|\.roo(/|code|$)|\.continue/|\.gemini|\.idea/|\.vscode/|copilot-instructions|claude\.md|agents\.md|mcp\.json|` + + `_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` + `credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json|py)$|bootstrap\.properties|` + `\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg|key|pem)$)`) bugText = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`) diff --git a/internal/triage/triage_test.go b/internal/triage/triage_test.go index aa9cfda..0be51c0 100644 --- a/internal/triage/triage_test.go +++ b/internal/triage/triage_test.go @@ -41,3 +41,16 @@ func TestProbeBeatsLifecycleFlags(t *testing.T) { t.Fatalf("got %s", got.Category) } } + +func TestAIToolConfigPathsAreProbes(t *testing.T) { + for _, p := range []string{"/.cursorrules", "/.codex/auth.json", "/backup/.claude/credentials.json", "/.windsurfrules", "/.vscode/launch.json", "/.idea/WebServers.xml", "/x/.github/copilot-instructions.md", "/CLAUDE.md", "/mcp.json", "/.clinerules"} { + if !IsProbePath(p) { + t.Errorf("%s should be a probe", p) + } + } + for _, p := range []string{"/mcp", "/api/agents", "/api/sessions/{id}", "/api/continue"} { + if IsProbePath(p) { + t.Errorf("%s must not be a probe", p) + } + } +} diff --git a/internal/uaclass/uaclass.go b/internal/uaclass/uaclass.go index 3af8402..d423df6 100644 --- a/internal/uaclass/uaclass.go +++ b/internal/uaclass/uaclass.go @@ -269,3 +269,14 @@ func IsBot(class string) bool { } return false } + +// IsCrawler reports self-declared crawlers, agents and scanners: the classes +// whose 4xx noise is not user impact. Monitors and plain HTTP libraries are +// excluded because their failures can be real. +func IsCrawler(class string) bool { + switch class { + case AICrawler, AIAgent, SearchCrawler, SEOCrawler, ScanTool, OtherBot: + return true + } + return false +} diff --git a/otel-ui/index.html b/otel-ui/index.html index ca36007..fb02994 100644 --- a/otel-ui/index.html +++ b/otel-ui/index.html @@ -4,7 +4,7 @@ - HyperScope OTel — Distributed Telemetry + OhhTel — Distributed Telemetry