include db for gitea build

cmd/alert-test/main.go

@@ -0,0 +1,43 @@
+// alert-test sends one clearly marked test alert through every channel
+// configured in the environment (see internal/alerts.ConfigFromEnv), so a new
+// Telegram token / chat id or webhook can be verified. It never prints secrets.
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/alerts"
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+func main() {
+	cfg := alerts.ConfigFromEnv()
+	notifiers := cfg.Notifiers()
+	if len(notifiers) == 0 {
+		fmt.Fprintln(os.Stderr, "no channel configured (TELEGRAM_KEY + TELEGRAM_CHATID, or OTEL_ALERT_WEBHOOK_URL)")
+		os.Exit(2)
+	}
+	now := time.Now().UTC()
+	a := alerts.NewAlert(features.Spike{Service: "alert-test", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 42, BaselineMean: 1, Excess: 41,
+		Patterns: []features.PatternDelta{{Signature: "test", Count: 42, BaselineMean: 1, Excess: 41, Novel: true, Clients: 3,
+			Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert from otel-debug"}}}}, now)
+	a.Test = true
+	failed := false
+	for _, n := range notifiers {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		err := n.Send(ctx, a)
+		cancel()
+		if err != nil {
+			fmt.Printf("%s: FAILED: %v\n", n.Name(), err)
+			failed = true
+			continue
+		}
+		fmt.Printf("%s: sent\n", n.Name())
+	}
+	if failed {
+		os.Exit(1)
+	}
+}

cmd/bad-actors/main.go

@@ -17,14 +17,19 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 
 	"git.tornberg.me/go-otel/internal/honeypot"
+	"git.tornberg.me/go-otel/internal/tracectx"
 	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
 )
 
 type otlp struct {
 	ResourceSpans []struct {
 		ScopeSpans []struct {
 			Spans []struct {
-				Attributes []struct {
+				TraceID      string `json:"traceId"`
+				SpanID       string `json:"spanId"`
+				ParentSpanID string `json:"parentSpanId"`
+				Attributes   []struct {
 					Key   string                     `json:"key"`
 					Value map[string]json.RawMessage `json:"value"`
 				} `json:"attributes"`
@@ -76,8 +81,15 @@ func main() {
 	}
 	defer rows.Close()
 
-	actors := map[string]*actor{}
-	spans := 0
+	// Pass 1 collects every span; pass 2 resolves parents, because the outer
+	// nginx span (real client) and the inner loopback span are exported in
+	// different envelopes and arrive in either order.
+	type record struct {
+		key                  tracectx.Key
+		ip, path, status, ua string
+	}
+	var records []record
+	var all []tracectx.Span
 	for rows.Next() {
 		var data string
 		if err := rows.Scan(&data); err != nil {
@@ -99,25 +111,9 @@ func main() {
 						}
 					}
 					ip := strings.TrimSpace(strings.SplitN(first(at, "http.request.header.x-real-ip", "http.request.header.x-forwarded-for", "client.address", "net.peer.ip"), ",", 2)[0])
-					if ip == "" {
-						continue
-					}
-					spans++
 					path := strings.SplitN(first(at, "http.target", "url.path", "http.route"), "?", 2)[0]
-					a := actors[ip]
-					if a == nil {
-						a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
-						actors[ip] = a
-					}
-					a.Total++
-					if a.UserAgent == "" {
-						a.UserAgent = first(at, "http.user_agent")
-					}
-					if triage.IsProbePath(path) {
-						a.Probes++
-						a.Paths[path] = struct{}{}
-						a.Statuses[first(at, "http.status_code")]++
-					}
+					all = append(all, tracectx.Span{TraceID: sp.TraceID, SpanID: sp.SpanID, ParentID: sp.ParentSpanID, Service: *service, PeerIP: ip, Edge: true})
+					records = append(records, record{key: tracectx.Key{TraceID: sp.TraceID, SpanID: sp.SpanID}, ip: ip, path: path, status: first(at, "http.status_code"), ua: first(at, "http.user_agent")})
 				}
 			}
 		}
@@ -126,6 +122,39 @@ func main() {
 		log.Fatal(err)
 	}
 
+	resolved := tracectx.Resolve(all)
+	actors := map[string]*actor{}
+	spans, recovered, recoveredProbes := 0, 0, 0
+	for _, r := range records {
+		ip := r.ip
+		if in, ok := resolved[r.key]; ok && in.ClientIP != "" && in.ClientHops > 0 && internal(ip) {
+			ip = in.ClientIP // hop behind a proxy: use the real client from an ancestor
+			recovered++
+			if triage.IsProbePath(r.path) {
+				recoveredProbes++
+			}
+		}
+		if ip == "" {
+			continue
+		}
+		spans++
+		a := actors[ip]
+		if a == nil {
+			a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
+			actors[ip] = a
+		}
+		a.Total++
+		if a.UserAgent == "" {
+			a.UserAgent = r.ua
+		}
+		if triage.IsProbePath(r.path) {
+			a.Probes++
+			a.Paths[r.path] = struct{}{}
+			a.Statuses[r.status]++
+		}
+	}
+	fmt.Printf("resolved %d hop spans to their real client via the parent span (%d of them probes)\n", recovered, recoveredProbes)
+
 	var list []*actor
 	totalProbes, blockableProbes, blockable, skippedInternal, skippedInternalProbes := 0, 0, 0, 0, 0
 	for _, a := range actors {
@@ -149,12 +178,12 @@ func main() {
 	fmt.Printf("%d spans with an IP, %d distinct IPs, %d probe requests\n", spans, len(actors), totalProbes)
 	fmt.Printf("blockable (public; >=%d probes/>=%d paths/>=%.0f%% probes, or >=%d probes over >=%d paths): %d IPs covering %d probes (%.0f%%)\n", *minProbes, *minPaths, *minShare*100, *broadProbes, *broadPaths, blockable, blockableProbes, pct(blockableProbes, totalProbes))
 	fmt.Printf("internal/loopback (never blocked): %d IPs, %d probes (%.0f%%) - real client is hidden behind a proxy hop\n\n", skippedInternal, skippedInternalProbes, pct(skippedInternalProbes, totalProbes))
-	fmt.Printf("%-40s %6s %6s %6s  %s\n", "ip", "probes", "total", "paths", "verdict / user-agent / statuses")
+	fmt.Printf("%-40s %6s %6s %6s  %-14s %s\n", "ip", "probes", "total", "paths", "class", "verdict / user-agent / statuses")
 	for i, a := range list {
 		if i >= *top {
 			break
 		}
-		fmt.Printf("%-40s %6d %6d %6d  %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), a.UserAgent, a.Statuses)
+		fmt.Printf("%-40s %6d %6d %6d  %-14s %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), uaclass.Of(strings.TrimPrefix(strings.TrimPrefix(a.UserAgent, "[BLOCK] "), "[INTERNAL] ")), a.UserAgent, a.Statuses)
 	}
 }
 

cmd/server/main.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"strings"
 
+	"git.tornberg.me/go-otel/internal/alerts"
 	"git.tornberg.me/go-otel/internal/auth"
 	"git.tornberg.me/go-otel/internal/collector"
 	"git.tornberg.me/go-otel/internal/geoip"
@@ -243,6 +244,12 @@ func main() {
 	if err != nil {
 		log.Printf("ip blocker disabled: %v", err)
 	}
+	// Optional spike alerts (Telegram / webhook): off unless a channel is
+	// configured or OTEL_ALERT_DRY_RUN=true.
+	var alertRunner *alerts.Runner
+	if chStore, ok := store.(*storage.ClickHouseStore); ok {
+		alertRunner = alerts.Start(context.Background(), alerts.ConfigFromEnv(), chStore.Conn(), chStore.Database())
+	}
 	// Update collector with wsServer
 	coll.SetBroadcaster(wsServer)
 	// Optional MaxMind databases for the UI's IP details dialog.
@@ -309,6 +316,10 @@ func main() {
 	uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig)
 	uiMux.Handle("GET /api/ip/{ip}", oidcAuth.RequireAuth(http.HandlerFunc(geo.Handler)))
 	uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser)))
+	if alertRunner != nil {
+		uiMux.Handle("GET /api/alerts", oidcAuth.RequireAuth(http.HandlerFunc(alertRunner.HandleStatus)))
+		uiMux.Handle("POST /api/alerts/test", oidcAuth.RequireAuth(http.HandlerFunc(alertRunner.HandleTest)))
+	}
 	if ipBlocker != nil {
 		uiMux.Handle("GET /api/blocker/status", oidcAuth.RequireAuth(http.HandlerFunc(ipBlocker.HandleStatus)))
 	}

cmd/spike-alerts/main.go

@@ -17,8 +17,10 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 
+	"git.tornberg.me/go-otel/internal/alerts"
 	"git.tornberg.me/go-otel/internal/features"
 	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
 )
 
 func main() {
@@ -51,18 +53,28 @@ func main() {
 	}
 	cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}
 
-	events := load(*addr, *database, *user, *password, start, end)
-	cfg.DataSince = dataSince(*addr, *database, *user, *password)
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	events, err := alerts.LoadErrorEvents(ctx, conn, *database, start, end)
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg.DataSince = alerts.DataSince(ctx, conn, *database)
 	log.Printf("loaded %d error events", len(events))
 
 	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
 	muted := map[triage.Category]int{}
-	alerts := 0
+	fired := 0
 	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
-		for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
-			alerts++
+		for _, spike := range suppress.Filter(alerts.FilterActionable(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
+			fired++
 			var cls []classification
 			for _, p := range spike.Patterns {
 				c := classification{}
@@ -86,7 +98,7 @@ func main() {
 	for c, n := range muted {
 		log.Printf("muted %d pattern-window(s) as %s", n, c)
 	}
-	log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
+	log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
 
 type classification struct {
@@ -96,35 +108,6 @@ type classification struct {
 	Err        string  `json:"error,omitempty"`
 }
 
-func triageOf(e features.Event) triage.Result {
-	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
-		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
-}
-
-// triageFilter keeps only actionable patterns (unless showMuted), recomputing
-// each spike's excess so a probe storm cannot page anyone by itself.
-func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
-	var out []features.Spike
-	for _, sp := range spikes {
-		var kept []features.PatternDelta
-		sp.Excess = 0
-		for _, p := range sp.Patterns {
-			c := triageOf(p.Example).Category
-			if !showMuted && !c.Actionable() {
-				muted[c]++
-				continue
-			}
-			kept = append(kept, p)
-			sp.Excess += p.Excess
-		}
-		if len(kept) > 0 {
-			sp.Patterns = kept
-			out = append(out, sp)
-		}
-	}
-	return out
-}
-
 func classify(base, text string) classification {
 	body, _ := json.Marshal(map[string]string{"text": text})
 	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
@@ -163,47 +146,19 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Novel {
 			tag = "NEW"
 		}
-		what := string(triageOf(p.Example).Category)
+		what := string(alerts.TriageOf(p.Example).Category)
 		if cls[i].Action != "" {
 			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
 		}
-		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
-	}
-}
-
-func load(addr, database, user, password string, start, end time.Time) []features.Event {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	defer cancel()
-	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer conn.Close()
-	// Errors only, and only columns present in every deployed schema version.
-	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
-		FROM %s.normalized_events FINAL
-		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
-		ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer rows.Close()
-	var out []features.Event
-	for rows.Next() {
-		e := features.Event{IsError: true}
-		var status uint16
-		var timeout, cancel, retry uint8
-		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
-			log.Fatal(err)
+		clients := ""
+		if p.Clients > 0 {
+			clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
 		}
-		e.StatusCode = int(status)
-		e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
-		out = append(out, e)
+		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
+			clients += " [" + p.Example.UAClass + "]"
+		}
+		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
 	}
-	if err := rows.Err(); err != nil {
-		log.Fatal(err)
-	}
-	return out
 }
 
 func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
@@ -213,23 +168,3 @@ func envOr(k, f string) string {
 	}
 	return f
 }
-
-// dataSince returns the earliest normalized row, used as the warm-up guard.
-func dataSince(addr, database, user, password string) time.Time {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-	defer cancel()
-	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer conn.Close()
-	var t time.Time
-	if err := conn.QueryRow(ctx, fmt.Sprintf(`SELECT min(timestamp) FROM %s.normalized_events`, database)).Scan(&t); err != nil {
-		log.Printf("warm-up guard disabled: %v", err)
-		return time.Time{}
-	}
-	if t.Year() < 2000 { // empty table
-		return time.Time{}
-	}
-	return t
-}

cmd/window-features/main.go

@@ -31,6 +31,12 @@ type event struct {
 	IsCancellation   uint8     `json:"isCancellation"`
 	Message          string    `json:"messageNormalized"`
 	SpanName         string    `json:"spanName"`
+	// Optional trace-resolved fields (internal/tracectx); absent in older exports.
+	ClientIP    string `json:"clientIp"`
+	UAClass     string `json:"uaClass"`
+	IsDuplicate uint8  `json:"isDuplicate"`
+	IsRootCause uint8  `json:"isRootCause"`
+	HasUpstream uint8  `json:"hasUpstream"`
 }
 
 func main() {
@@ -95,6 +101,7 @@ func readEvents(path string) []features.Event {
 			IsError: item.IsError != 0, StatusClass: item.StatusClass, DurationMS: item.DurationMS, TraceID: item.TraceID,
 			PatternSignature: item.PatternSignature, IssueFingerprint: item.IssueFingerprint, IsRetry: item.IsRetry != 0,
 			IsTimeout: item.IsTimeout != 0, IsCancellation: item.IsCancellation != 0, Message: item.Message, SpanName: item.SpanName,
+			ClientIP: item.ClientIP, UAClass: item.UAClass, IsDuplicate: item.IsDuplicate != 0, IsRootCause: item.IsRootCause != 0, HasUpstream: item.HasUpstream != 0,
 		})
 	}
 	if err := scanner.Err(); err != nil {