more

internal/collector/handlers.go

@@ -852,6 +852,41 @@ func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request)
 	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
 }
 
+// HandleStatsEndpoints returns normalized HTTP traffic groups. It is useful
+// for diagnosing noisy hosts and proxy/instrumentation duplication without
+// loading raw telemetry into the browser.
+func (c *Collector) HandleStatsEndpoints(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+	service, hours := parseStatsCommon(r)
+	host := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("host")))
+	limit := parseLimit(r, 100)
+	if limit <= 0 {
+		limit = 100
+	}
+	if limit > 500 {
+		limit = 500
+	}
+	stats, err := c.statsQuerier.EndpointStats(r.Context(), host, service, hours, limit)
+	if err != nil {
+		log.Printf("failed to compute endpoint stats: %v", err)
+		http.Error(w, "failed to compute endpoint stats", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"hours":     hours,
+		"host":      host,
+		"endpoints": stats,
+	})
+}
+
 // needleAddr returns the configured or default address for the Needle 3 sidecar.
 func needleAddr() string {
 	addr := os.Getenv("OTEL_NEEDLE_ADDR")

internal/normalization/normalizer.go

@@ -26,6 +26,13 @@ type Event struct {
 	Service           string  `json:"service"`
 	Release           string  `json:"release,omitempty"`
 	Environment       string  `json:"environment,omitempty"`
+	Method            string  `json:"method,omitempty"`
+	Host              string  `json:"host,omitempty"`
+	UserAgent         string  `json:"userAgent,omitempty"`
+	OriginHost        string  `json:"originHost,omitempty"`
+	RefererHost       string  `json:"refererHost,omitempty"`
+	FetchSite         string  `json:"fetchSite,omitempty"`
+	FetchMode         string  `json:"fetchMode,omitempty"`
 	Kind              string  `json:"kind"`
 	Source            string  `json:"source"`
 	IsError           bool    `json:"isError"`
@@ -108,7 +115,7 @@ func normalizeTraces(ingest int64, envelope map[string]json.RawMessage) []Event
 				if exceptionType != "" && message != "" {
 					message = exceptionType + ": " + message
 				}
-				isError := status == 2 || statusCode >= 400 || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != ""
+				isError := status == 2 || isErrorStatus(statusCode) || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != ""
 				e := baseEvent(ingest, attrs, "span", item.TraceID)
 				e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode)
 				e.DurationMS = durationMS(item.Start, item.End)
@@ -140,7 +147,7 @@ func normalizeLogs(ingest int64, envelope map[string]json.RawMessage) []Event {
 				statusCode := intAttr(attrs, "http.response.status_code", "http.status_code")
 				grpcCode := intAttr(attrs, "rpc.grpc.status_code")
 				e := baseEvent(timestampOr(ingest, item.Timestamp), attrs, "log", item.TraceID)
-				e.IsError = item.SeverityNumber >= 17 || strings.EqualFold(item.SeverityText, "ERROR") || strings.EqualFold(item.SeverityText, "FATAL") || exceptionType != "" || statusCode >= 400 || grpcCode > 0
+				e.IsError = item.SeverityNumber >= 17 || strings.EqualFold(item.SeverityText, "ERROR") || strings.EqualFold(item.SeverityText, "FATAL") || exceptionType != "" || isErrorStatus(statusCode) || grpcCode > 0
 				e.StatusCode, e.StatusClass = statusCode, statusClass(statusCode)
 				e.Route, e.ExceptionType, e.MessageNormalized, e.StackSignature = normalizeHTTPRoute(attrs), exceptionType, normalizeMessage(message), firstFrame(stack)
 				e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, grpcCode)
@@ -301,7 +308,28 @@ func baseEvent(timestamp int64, attrs []attribute, kind, traceID string) Event {
 	if source == "" {
 		source = "backend"
 	}
-	return Event{Timestamp: timestamp, NormalizerVersion: Version, Service: defaultString(stringAttr(attrs, "service.name"), "unknown"), Release: stringAttr(attrs, "service.release"), Environment: defaultString(stringAttr(attrs, "deployment.environment.name", "deployment.environment", "service.environment"), ""), Kind: kind, Source: source, TraceID: traceID}
+	ep := httpattr.Extract(func(keys ...string) string { return stringAttr(attrs, keys...) })
+	return Event{
+		Timestamp: timestamp, NormalizerVersion: Version,
+		Service:     defaultString(stringAttr(attrs, "service.name"), "unknown"),
+		Release:     stringAttr(attrs, "service.release"),
+		Environment: defaultString(stringAttr(attrs, "deployment.environment.name", "deployment.environment", "service.environment"), ""),
+		Method:      ep.Method, Host: ep.Host,
+		UserAgent:   stringAttr(attrs, "http.user_agent", "user_agent.original"),
+		OriginHost:  attrHost(stringAttr(attrs, "http.request.header.origin", "http.origin", "origin")),
+		RefererHost: attrHost(stringAttr(attrs, "http.request.header.referer", "http.referer")),
+		FetchSite:   stringAttr(attrs, "http.request.header.sec_fetch_site"),
+		FetchMode:   stringAttr(attrs, "http.request.header.sec_fetch_mode"),
+		Kind:        kind, Source: source, TraceID: traceID,
+	}
+}
+
+func attrHost(raw string) string {
+	if raw == "" {
+		return ""
+	}
+	host, _ := httpattr.SplitURL(raw)
+	return host
 }
 func finish(e Event) Event {
 	if e.IsError {
@@ -445,3 +473,10 @@ func exception(events []spanEvent) (string, string, string) {
 	}
 	return "", "", ""
 }
+
+// isErrorStatus reports whether an HTTP status alone marks an event as an
+// error. 404/410 are excluded: missing resources and scanner probes would
+// otherwise dominate the error rate the spike label is built from.
+func isErrorStatus(code int) bool {
+	return code >= 400 && code != 404 && code != 410
+}

internal/normalization/normalizer_test.go

@@ -106,3 +106,29 @@ func TestNormalizeHTTPRoutePrefersSpecificPathOverRootMount(t *testing.T) {
 		t.Fatalf("expected specific URL path, got %q", events[0].Route)
 	}
 }
+
+func TestNormalizeHTTPRequestMetadataIsSafeAndNormalized(t *testing.T) {
+	data := []byte(`{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"preview"}}]},"scopeSpans":[{"spans":[{"traceId":"trace-1","name":"GET /","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.host","value":{"stringValue":"preview.example.test:443"}},{"key":"http.target","value":{"stringValue":"/"}},{"key":"http.user_agent","value":{"stringValue":"Chrome"}},{"key":"http.request.header.origin","value":{"stringValue":"https://app.example.test"}},{"key":"http.request.header.referer","value":{"stringValue":"https://app.example.test/dashboard?secret=1"}},{"key":"http.request.header.sec_fetch_site","value":{"stringValue":"same-origin"}},{"key":"http.request.header.sec_fetch_mode","value":{"stringValue":"navigate"}}]}]}]}]}`)
+	events, err := NormalizeEntry("traces", 99, data)
+	if err != nil || len(events) != 1 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	event := events[0]
+	if event.Method != "GET" || event.Host != "preview.example.test" || event.UserAgent != "Chrome" {
+		t.Fatalf("unexpected request metadata: %+v", event)
+	}
+	if event.OriginHost != "app.example.test" || event.RefererHost != "app.example.test" {
+		t.Fatalf("request metadata must retain hosts without paths: %+v", event)
+	}
+	if event.FetchSite != "same-origin" || event.FetchMode != "navigate" {
+		t.Fatalf("missing fetch metadata: %+v", event)
+	}
+}
+
+func TestIsErrorStatusIgnoresMissingResources(t *testing.T) {
+	for code, want := range map[int]bool{200: false, 404: false, 410: false, 401: true, 500: true} {
+		if got := isErrorStatus(code); got != want {
+			t.Errorf("isErrorStatus(%d) = %v, want %v", code, got, want)
+		}
+	}
+}

internal/storage/clickhouse_endpoints.go

@@ -0,0 +1,94 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// EndpointStats reads raw trace spans because the production entries table is
+// the source of truth and may not have the optional normalized_events table.
+// The query intentionally returns both span and trace counts: a reverse proxy
+// can legitimately produce nested spans for one request, as with nginx
+// ingress -> upstream instrumentation.
+func (s *ClickHouseStore) EndpointStats(ctx context.Context, host, service string, hours, limit int) ([]EndpointStat, error) {
+	if hours <= 0 {
+		hours = 72
+	}
+	if limit <= 0 || limit > 500 {
+		limit = 100
+	}
+
+	where := "host != ''"
+	rawWhere := "type = 'traces' AND timestamp >= ?"
+	args := []interface{}{time.Now().UTC().Add(-time.Duration(hours) * time.Hour)}
+	if host != "" {
+		where += " AND host = ?"
+		args = append(args, host)
+		rawWhere += " AND positionCaseInsensitive(data, ?) > 0"
+		args = append(args, host)
+	}
+	if service != "" {
+		where += " AND service = ?"
+		args = append(args, service)
+	}
+
+	query := fmt.Sprintf(`
+		SELECT service, host, method, path, if(route_raw = '/' AND path != '/', '', route_raw) AS route, status_code, user_agent,
+			origin_host, referer_host, fetch_site, fetch_mode,
+			count() AS span_count, uniqExact(trace_id) AS trace_count,
+			min(timestamp) AS first_seen, max(timestamp) AS last_seen
+		FROM (
+			WITH JSONExtractArrayRaw(span, 'attributes') AS attrs
+			SELECT
+				timestamp,
+				JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'service.name', JSONExtractArrayRaw(JSONExtractRaw(resource_span, 'resource'), 'attributes')), 'value', 'stringValue') AS service,
+				JSONExtractString(span, 'traceId') AS trace_id,
+				upper(coalesce(nullIf(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.request.method', 'http.method'), attrs), 'value', 'stringValue'), ''), '')) AS method,
+				replaceRegexpOne(coalesce(nullIf(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'url.path', attrs), 'value', 'stringValue'), ''), JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'http.target', attrs), 'value', 'stringValue')), '[?#].*$', '') AS path,
+				JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'http.route', attrs), 'value', 'stringValue') AS route_raw,
+				lower(replaceRegexpOne(coalesce(nullIf(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.host', 'server.address', 'net.host.name'), attrs), 'value', 'stringValue'), ''), ''), ':\\d+$', '')) AS host,
+				toUInt16OrZero(coalesce(nullIf(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.response.status_code', 'http.status_code'), attrs), 'value', 'intValue'), ''), JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.response.status_code', 'http.status_code'), attrs), 'value', 'stringValue'))) AS status_code,
+				JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.user_agent', 'user_agent.original'), attrs), 'value', 'stringValue') AS user_agent,
+				replaceRegexpOne(replaceRegexpOne(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.request.header.origin', 'http.origin', 'origin'), attrs), 'value', 'stringValue'), '^https?://', ''), '/.*$', '') AS origin_host,
+				replaceRegexpOne(replaceRegexpOne(JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') IN ('http.request.header.referer', 'http.referer'), attrs), 'value', 'stringValue'), '^https?://', ''), '/.*$', '') AS referer_host,
+				JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'http.request.header.sec_fetch_site', attrs), 'value', 'stringValue') AS fetch_site,
+				JSONExtractString(arrayFirst(x -> JSONExtractString(x, 'key') = 'http.request.header.sec_fetch_mode', attrs), 'value', 'stringValue') AS fetch_mode
+			FROM (
+				SELECT timestamp, span
+				FROM %s.entries
+				ARRAY JOIN JSONExtractArrayRaw(data, 'resourceSpans') AS resource_span
+				ARRAY JOIN JSONExtractArrayRaw(resource_span, 'scopeSpans') AS scope_span
+				ARRAY JOIN JSONExtractArrayRaw(scope_span, 'spans') AS span
+				WHERE %s
+			)
+		) AS spans
+		WHERE %s
+		GROUP BY service, host, method, path, route_raw, status_code, user_agent, origin_host, referer_host, fetch_site, fetch_mode
+		ORDER BY trace_count DESC, span_count DESC, last_seen DESC
+		LIMIT ?`, s.db, rawWhere, where)
+	args = append(args, limit)
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse endpoint stats: %w", err)
+	}
+	defer rows.Close()
+
+	var out []EndpointStat
+	for rows.Next() {
+		var stat EndpointStat
+		if err := rows.Scan(
+			&stat.Service, &stat.Host, &stat.Method, &stat.Path, &stat.Route, &stat.StatusCode,
+			&stat.UserAgent, &stat.OriginHost, &stat.RefererHost, &stat.FetchSite,
+			&stat.FetchMode, &stat.SpanCount, &stat.TraceCount, &stat.FirstSeen, &stat.LastSeen,
+		); err != nil {
+			return nil, err
+		}
+		if stat.Route == "/" && stat.Path != "/" {
+			stat.Route = ""
+		}
+		out = append(out, stat)
+	}
+	return out, rows.Err()
+}

internal/storage/store.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"git.tornberg.me/go-otel/internal/websocket"
+	"time"
 )
 
 // SearchOptions narrows a Search call beyond the free-text query.
@@ -38,6 +39,31 @@ type StatsQuerier interface {
 	ReleaseSeries(ctx context.Context, service string, hours int) (map[string][]StatsPoint, error)
 	// Services lists distinct services seen in the rollup.
 	Services(ctx context.Context) ([]string, error)
+	// EndpointStats groups HTTP spans by normalized request identity and safe
+	// browser metadata. SpanCount exposes proxy/instrumentation duplication;
+	// TraceCount is the better approximation of logical requests.
+	EndpointStats(ctx context.Context, host, service string, hours, limit int) ([]EndpointStat, error)
+}
+
+// EndpointStat is a bounded, privacy-conscious HTTP traffic aggregate. Origin
+// and referer are host-only values; paths and query strings are normalized by
+// the query so secrets are not returned as part of endpoint diagnostics.
+type EndpointStat struct {
+	Service     string    `json:"service"`
+	Host        string    `json:"host"`
+	Method      string    `json:"method"`
+	Path        string    `json:"path"`
+	Route       string    `json:"route,omitempty"`
+	StatusCode  uint16    `json:"statusCode"`
+	UserAgent   string    `json:"userAgent,omitempty"`
+	OriginHost  string    `json:"originHost,omitempty"`
+	RefererHost string    `json:"refererHost,omitempty"`
+	FetchSite   string    `json:"fetchSite,omitempty"`
+	FetchMode   string    `json:"fetchMode,omitempty"`
+	SpanCount   uint64    `json:"spanCount"`
+	TraceCount  uint64    `json:"traceCount"`
+	FirstSeen   time.Time `json:"firstSeen"`
+	LastSeen    time.Time `json:"lastSeen"`
 }
 
 // TraceSummary is the per-trace rollup shown in overview tables: everything