ml

internal/collector/collector_test.go

@@ -392,6 +392,31 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestHandleMLDetectorTest(t *testing.T) {
+	coll := New(nil, 10, nil)
+	req := httptest.NewRequest(http.MethodPost, "/api/ml/test-spike", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleMLDetectorTest(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected detector test success, got %d: %s", rec.Code, rec.Body.String())
+	}
+	var result struct {
+		Passed bool `json:"passed"`
+		Row    struct {
+			LabelKnown        bool    `json:"labelKnown"`
+			Label             bool    `json:"label"`
+			BaselineErrorRate float64 `json:"baselineErrorRate"`
+			FutureErrorCount  int     `json:"futureErrorCount"`
+		} `json:"row"`
+	}
+	if err := json.NewDecoder(rec.Body).Decode(&result); err != nil {
+		t.Fatal(err)
+	}
+	if !result.Passed || !result.Row.LabelKnown || !result.Row.Label || result.Row.BaselineErrorRate != .1 || result.Row.FutureErrorCount != 5 {
+		t.Fatalf("unexpected detector verification response: %+v", result)
+	}
+}
+
 func TestHandleIssuesFiltersAndPaginates(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 	coll.issues.Record("span_error", "nginx", "GET /.env", issues.Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})

internal/collector/handlers.go

@@ -10,10 +10,12 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/features"
 	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
@@ -669,6 +671,41 @@ func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// HandleMLDetectorTest runs a deterministic, in-memory spike through the
+// production feature and label builder. It stores no telemetry and exists so
+// an operator can verify the detector wiring from the authenticated UI.
+func (c *Collector) HandleMLDetectorTest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	const service = "ml-verification"
+	base := time.Date(2026, 1, 8, 12, 0, 0, 0, time.UTC)
+	events := make([]features.Event, 0, 100)
+	for day := 1; day <= 7; day++ {
+		for i := 0; i < 10; i++ {
+			events = append(events, features.Event{Timestamp: base.AddDate(0, 0, -day).Add(time.Duration(i) * time.Second), Service: service, Kind: "span", IsError: i == 0, DurationMS: float64(i + 1)})
+		}
+	}
+	for i := 0; i < 10; i++ {
+		events = append(events, features.Event{Timestamp: base.Add(time.Duration(i) * time.Second), Service: service, Release: "verification", Kind: "span", DurationMS: float64(i + 1)})
+		events = append(events, features.Event{Timestamp: base.Add(5 * time.Minute).Add(time.Duration(i) * time.Second), Service: service, Release: "verification", Kind: "log", IsError: i < 5, IssueFingerprint: "verification-spike"})
+	}
+	rows := features.Build(events, features.Config{ObservedUntil: base.Add(20 * time.Minute)})
+	for _, row := range rows {
+		if row.WindowStart.Equal(base) && row.Service == service && row.Release == "verification" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]interface{}{
+				"passed":   row.LabelKnown && row.Label,
+				"scenario": "five errors in ten future events against a seven-day 10% baseline",
+				"row":      row,
+			})
+			return
+		}
+	}
+	jsonError(w, http.StatusInternalServerError, "detector test did not generate its verification row")
+}
+
 // statsEnabled reports whether an aggregate-capable store is configured.
 func (c *Collector) statsEnabled() bool { return c.statsQuerier != nil }
 
@@ -767,3 +804,107 @@ func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request)
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
 }
+
+// needleAddr returns the configured or default address for the Needle 3 sidecar.
+func needleAddr() string {
+	addr := os.Getenv("OTEL_NEEDLE_ADDR")
+	if addr == "" {
+		addr = "http://127.0.0.1:5005"
+	}
+	return strings.TrimRight(addr, "/")
+}
+
+// HandleNeedleStatus checks if the Needle 3 sidecar is reachable and reports its health.
+func (c *Collector) HandleNeedleStatus(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	target := needleAddr() + "/health"
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(target)
+	w.Header().Set("Content-Type", "application/json")
+	if err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"available": false,
+			"error":     fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
+		})
+		return
+	}
+	defer resp.Body.Close()
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// HandleNeedleClassify proxies telemetry triage requests to the Needle 3 sidecar.
+func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "failed to read request body", http.StatusBadRequest)
+		return
+	}
+
+	// If the payload specifies an issue_id and no text, look up the issue
+	var reqMap map[string]interface{}
+	if err := json.Unmarshal(body, &reqMap); err == nil {
+		if issueID, ok := reqMap["issue_id"].(string); ok && issueID != "" {
+			if issue, ok := c.GetIssue(issueID); ok {
+				var promptParts []string
+				promptParts = append(promptParts, fmt.Sprintf("Service: %s", issue.Service))
+				promptParts = append(promptParts, fmt.Sprintf("Title: %s", issue.Title))
+				promptParts = append(promptParts, fmt.Sprintf("Kind: %s", issue.Kind))
+				if len(issue.Samples) > 0 {
+					latest := issue.Samples[len(issue.Samples)-1]
+					if latest.Message != "" {
+						promptParts = append(promptParts, fmt.Sprintf("Message: %s", latest.Message))
+					}
+					if latest.URL != "" {
+						promptParts = append(promptParts, fmt.Sprintf("URL: %s", latest.URL))
+					}
+					if latest.HTTPStatus > 0 {
+						promptParts = append(promptParts, fmt.Sprintf("Status: %d", latest.HTTPStatus))
+					}
+					if latest.HTTPRoute != "" {
+						promptParts = append(promptParts, fmt.Sprintf("Route: %s", latest.HTTPRoute))
+					}
+				}
+				reqMap["text"] = strings.Join(promptParts, " | ")
+				reqMap["service"] = issue.Service
+				reqMap["title"] = issue.Title
+				body, _ = json.Marshal(reqMap)
+			}
+		}
+	}
+
+	target := needleAddr() + "/classify"
+	client := &http.Client{Timeout: 60 * time.Second}
+	proxyReq, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target, strings.NewReader(string(body)))
+	if err != nil {
+		http.Error(w, "failed to create proxy request", http.StatusInternalServerError)
+		return
+	}
+	proxyReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(proxyReq)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"success": false,
+			"error":   fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
+		})
+		return
+	}
+	defer resp.Body.Close()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}

internal/collector/issue_extract.go

@@ -87,7 +87,7 @@ func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title s
 		SpanID:      idString(span.SpanID()),
 		Message:     title,
 		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
-		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		HTTPRoute:   httpRoute(span.Attributes()),
 		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
 		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
 		Release:     firstAttr(span.Attributes(), "service.release"),
@@ -109,7 +109,7 @@ func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, tit
 		Message:     title,
 		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
 		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
-		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		HTTPRoute:   httpRoute(record.Attributes()),
 		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
 		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
 		Release:     firstAttr(record.Attributes(), "service.release"),
@@ -179,6 +179,18 @@ func firstAttr(attrs pcommon.Map, keys ...string) string {
 	return ""
 }
 
+// httpRoute prefers the concrete URL path when instrumentation reports "/"
+// as a generic router mount. This keeps API endpoints distinct in issue
+// fingerprints and avoids collapsing unrelated requests into one root route.
+func httpRoute(attrs pcommon.Map) string {
+	route := firstAttr(attrs, "http.route")
+	path := firstAttr(attrs, "url.path", "http.target")
+	if path != "" && (route == "" || route == "/" && path != "/") {
+		return path
+	}
+	return route
+}
+
 // getHTTPStatusCode extracts an HTTP status code from attributes, supporting
 // both new (http.response.status_code) and old (http.status_code) semconv keys,
 // stored as integer or string.