alot more changes
Build and Publish / Metadata (push) Successful in 8s
Build and Publish / BuildAndDeployAmd64 (push) Failing after 36s

This commit is contained in:
mats committed 2026-09-21 18:59:13 +02:00
1 parent 23243ddbdc
commit 0ed2935f04
40 files changed
+2889 -115

No files matched your search

+52 -3
View File
@@ -22,6 +22,7 @@ import (
"git.tornberg.me/go-otel/internal/websocket"
"go.opentelemetry.io/collector/consumer"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/plog"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.opentelemetry.io/collector/pdata/ptrace"
@@ -42,6 +43,7 @@ type Collector struct {
ingestLimiter *ipRateLimiter
ingestKey string
sourceMapKey string
notifyIssuesFn func()
@@ -49,6 +51,7 @@ type Collector struct {
// The stats endpoints return 501 when it's nil so the UI can hide those
// views instead of erroring.
statsQuerier storage.StatsQuerier
sourceIPs storage.SourceIPQuerier
sourceMaps *sourcemaps.Resolver
kubeWatcher *kube.Watcher
prometheus *prometheus.Client
@@ -111,6 +114,9 @@ func (c *Collector) SetIngestKey(key string) {
c.ingestKey = key
}
// SetSourceMapKey sets the shared key required to upload source maps.
func (c *Collector) SetSourceMapKey(key string) { c.sourceMapKey = key }
// HoneypotRecorder receives suspicious request observations extracted from
// telemetry. It is intentionally a small interface so the collector does not
// depend on the HTTP honeypot implementation.
@@ -390,7 +396,9 @@ func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
Data: buf,
}
c.broadcaster.Broadcast(&entry)
if c.broadcaster != nil {
c.broadcaster.Broadcast(&entry)
}
return nil
}
@@ -411,7 +419,9 @@ func (c *Collector) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro
Data: buf,
}
c.broadcaster.Broadcast(&entry)
if c.broadcaster != nil {
c.broadcaster.Broadcast(&entry)
}
return nil
}
@@ -436,11 +446,50 @@ func (c *Collector) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
Data: buf,
}
c.broadcaster.Broadcast(&entry)
if c.broadcaster != nil {
c.broadcaster.Broadcast(&entry)
}
return nil
}
// EmitInternalLog records a diagnostic generated by this server as an OTLP
// log. Keeping these in the same pipeline as customer telemetry makes deploy
// and ingestion problems visible in the UI's Logs view. Values are deliberately
// limited to scalar attributes so callers cannot accidentally attach payloads.
func (c *Collector) EmitInternalLog(ctx context.Context, severity plog.SeverityNumber, body string, attrs map[string]any, traceID pcommon.TraceID, spanID pcommon.SpanID) error {
logs := plog.NewLogs()
rl := logs.ResourceLogs().AppendEmpty()
rl.Resource().Attributes().PutStr("service.name", "go-otel")
rl.Resource().Attributes().PutStr("telemetry.sdk.name", "go-otel-internal")
sl := rl.ScopeLogs().AppendEmpty()
sl.Scope().SetName("git.tornberg.me/go-otel/internal")
record := sl.LogRecords().AppendEmpty()
record.SetTimestamp(pcommon.NewTimestampFromTime(time.Now()))
record.SetObservedTimestamp(pcommon.NewTimestampFromTime(time.Now()))
record.SetSeverityNumber(severity)
record.Body().SetStr(body)
if !traceID.IsEmpty() {
record.SetTraceID(traceID)
}
if !spanID.IsEmpty() {
record.SetSpanID(spanID)
}
for key, value := range attrs {
switch value := value.(type) {
case string:
record.Attributes().PutStr(key, value)
case int:
record.Attributes().PutInt(key, int64(value))
case int64:
record.Attributes().PutInt(key, value)
case bool:
record.Attributes().PutBool(key, value)
}
}
return c.ConsumeLogs(ctx, logs)
}
// Capabilities returns the capabilities of the collector.
func (c *Collector) Capabilities() consumer.Capabilities {
return consumer.Capabilities{MutatesData: false}
+81
View File
@@ -383,6 +383,21 @@ func TestHandleIssuesAndByID(t *testing.T) {
t.Fatalf("expected 200 for known issue id, got %d", rec.Code)
}
req = httptest.NewRequest(http.MethodPatch, "/api/issues/"+id, bytes.NewBufferString(`{"status":"resolved"}`))
req.SetPathValue("id", id)
rec = httptest.NewRecorder()
coll.HandleIssueByID(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 when resolving known issue, got %d: %s", rec.Code, rec.Body.String())
}
var updated map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &updated); err != nil {
t.Fatalf("failed to decode updated issue response: %v", err)
}
if updated["status"] != "resolved" {
t.Fatalf("expected resolved issue, got %+v", updated["status"])
}
req = httptest.NewRequest(http.MethodGet, "/api/issues/missing", nil)
req.SetPathValue("id", "missing")
rec = httptest.NewRecorder()
@@ -530,6 +545,72 @@ func TestHandleLogsIngestKey(t *testing.T) {
}
}
func TestHandleLogsPodGuardianIntegration(t *testing.T) {
coll := New(noopBroadcaster{}, 10, nil)
coll.SetIngestKey("ingest-secret")
// Payload matching what fervent-franklin (k8s-pod-guardian) sends
payload := `{
"resourceLogs": [
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "payment-service" } },
{ "key": "service.environment", "value": { "stringValue": "production" } },
{ "key": "telemetry.source", "value": { "stringValue": "k8s-pod-guardian" } },
{ "key": "k8s.namespace.name", "value": { "stringValue": "default" } },
{ "key": "k8s.pod.name", "value": { "stringValue": "payment-api-6789-xyz" } },
{ "key": "k8s.container.name", "value": { "stringValue": "payment-api" } },
{ "key": "host.name", "value": { "stringValue": "worker-1" } }
]
},
"scopeLogs": [
{
"scope": { "name": "k8s-pod-guardian/crashloop-harvester" },
"logRecords": [
{
"timeUnixNano": "1779362400000000000",
"severityNumber": 17,
"severityText": "ERROR",
"body": {
"stringValue": "panic: database connection failed: dial tcp 10.96.0.10:5432: i/o timeout"
},
"attributes": [
{ "key": "k8s.container.restart_count", "value": { "stringValue": "5" } },
{ "key": "crashloop.reason", "value": { "stringValue": "CrashLoopBackOff" } }
]
}
]
}
]
}
]
}`
req := httptest.NewRequest(http.MethodPost, "/v1/logs", bytes.NewReader([]byte(payload)))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Otel-Ingest-Key", "ingest-secret")
req.RemoteAddr = "192.0.2.1:12345"
rec := httptest.NewRecorder()
coll.HandleLogs(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d (body=%s)", rec.Code, rec.Body.String())
}
issues := coll.ListIssues(10)
if len(issues) != 1 {
t.Fatalf("expected 1 issue created from crash log, got %d", len(issues))
}
if issues[0].Service != "payment-service" {
t.Errorf("expected issue service 'payment-service', got %q", issues[0].Service)
}
if !strings.Contains(issues[0].Title, "panic: database connection failed") {
t.Errorf("unexpected issue title: %q", issues[0].Title)
}
}
func TestHandleLogsIngestRateLimited(t *testing.T) {
coll := New(noopBroadcaster{}, 10, nil)
body := []byte(`{"resourceLogs":[]}`)
+119 -1
View File
@@ -4,7 +4,9 @@ package collector
import (
"context"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"git.tornberg.me/go-otel/internal/sourcemaps"
@@ -20,6 +22,7 @@ import (
"git.tornberg.me/go-otel/internal/issues"
"git.tornberg.me/go-otel/internal/storage"
"git.tornberg.me/go-otel/internal/websocket"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/plog"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.opentelemetry.io/collector/pdata/ptrace"
@@ -477,32 +480,116 @@ func encodeIssueCursor(lastSeen int64, id string) string {
// HandleSourceMap accepts a JavaScript source map for a deployed release.
// Multipart fields: release, source, file.
func (c *Collector) HandleSourceMap(w http.ResponseWriter, r *http.Request) {
traceID, spanID := traceContextFromRequest(r)
logUpload := func(severity plog.SeverityNumber, body string, attrs map[string]any) {
if err := c.EmitInternalLog(r.Context(), severity, body, attrs, traceID, spanID); err != nil {
log.Printf("failed to emit sourcemap diagnostic: %v", err)
}
}
logUpload(plog.SeverityNumberInfo, "sourcemap upload received", map[string]any{
"event.name": "sourcemap.upload",
"http.route": "/api/sourcemaps",
"http.request.method": r.Method,
})
if r.Method != http.MethodPost {
logUpload(plog.SeverityNumberWarn, "sourcemap upload rejected: method not allowed", map[string]any{
"event.name": "sourcemap.upload.rejected",
"sourcemap.failure_stage": "method",
})
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if !c.checkSourceMapKey(r) {
logUpload(plog.SeverityNumberWarn, "sourcemap upload rejected: invalid or missing key", map[string]any{
"event.name": "sourcemap.upload.rejected",
"sourcemap.failure_stage": "auth",
"client.address": clientIP(r),
})
http.Error(w, "invalid or missing sourcemap key", http.StatusUnauthorized)
return
}
if c.sourceMaps == nil {
logUpload(plog.SeverityNumberError, "sourcemap upload failed: resolver is not configured", map[string]any{
"event.name": "sourcemap.upload.failed",
"sourcemap.failure_stage": "configuration",
})
http.Error(w, "source maps are not configured", http.StatusNotImplemented)
return
}
if err := r.ParseMultipartForm(50 << 20); err != nil {
logUpload(plog.SeverityNumberError, "sourcemap upload failed: invalid multipart body", map[string]any{
"event.name": "sourcemap.upload.failed",
"sourcemap.failure_stage": "multipart",
"error.type": "multipart_parse",
})
http.Error(w, "invalid multipart body", http.StatusBadRequest)
return
}
release, source := r.FormValue("release"), r.FormValue("source")
file, _, err := r.FormFile("file")
file, fileHeader, err := r.FormFile("file")
if err != nil {
logUpload(plog.SeverityNumberError, "sourcemap upload failed: file is missing", map[string]any{
"event.name": "sourcemap.upload.failed",
"sourcemap.failure_stage": "file",
"sourcemap.release": release,
"sourcemap.source": source,
"error.type": "missing_file",
})
http.Error(w, "file is required", http.StatusBadRequest)
return
}
defer file.Close()
if err := c.sourceMaps.Upload(release, source, file); err != nil {
logUpload(plog.SeverityNumberError, "sourcemap upload failed", map[string]any{
"event.name": "sourcemap.upload.failed",
"sourcemap.failure_stage": "store",
"sourcemap.release": release,
"sourcemap.source": source,
"sourcemap.size_bytes": fileHeader.Size,
"error.type": "source_map_invalid_or_unwritable",
"error.message": err.Error(),
})
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
logUpload(plog.SeverityNumberInfo, "sourcemap upload stored", map[string]any{
"event.name": "sourcemap.upload.stored",
"sourcemap.release": release,
"sourcemap.source": source,
"sourcemap.size_bytes": fileHeader.Size,
})
w.WriteHeader(http.StatusCreated)
}
// checkSourceMapKey authenticates a source map upload against the shared
// OTEL_SOURCEMAP_KEY (X-Sourcemap-Key header). Uploads are refused while no key
// is configured so the endpoint is never open by accident.
func (c *Collector) checkSourceMapKey(r *http.Request) bool {
if c.sourceMapKey == "" {
return false
}
got := r.Header.Get("X-Sourcemap-Key")
return subtle.ConstantTimeCompare([]byte(got), []byte(c.sourceMapKey)) == 1
}
// traceContextFromRequest preserves the caller's trace linkage for internal
// logs when the upload came through an instrumented client or proxy.
func traceContextFromRequest(r *http.Request) (pcommon.TraceID, pcommon.SpanID) {
var traceID pcommon.TraceID
var spanID pcommon.SpanID
parts := strings.Split(r.Header.Get("Traceparent"), "-")
if len(parts) != 4 || len(parts[1]) != 32 || len(parts[2]) != 16 {
return traceID, spanID
}
if _, err := hex.Decode(traceID[:], []byte(parts[1])); err != nil {
return pcommon.NewTraceIDEmpty(), pcommon.NewSpanIDEmpty()
}
if _, err := hex.Decode(spanID[:], []byte(parts[2])); err != nil {
return pcommon.NewTraceIDEmpty(), pcommon.NewSpanIDEmpty()
}
return traceID, spanID
}
// HandleListSourceMaps lists the release/source pairs currently loaded, so a
// deploy can verify its uploads (and that they survived a restart).
func (c *Collector) HandleListSourceMaps(w http.ResponseWriter, r *http.Request) {
@@ -518,6 +605,18 @@ func (c *Collector) HandleListSourceMaps(w http.ResponseWriter, r *http.Request)
// uploaded source maps. Body: {"items":[{"release":"...","stacktrace":"..."}]};
// the response has one resolved stacktrace per item, in order.
func (c *Collector) HandleResolveStacks(w http.ResponseWriter, r *http.Request) {
traceID, spanID := traceContextFromRequest(r)
logResolve := func(severity plog.SeverityNumber, body string, attrs map[string]any) {
if err := c.EmitInternalLog(r.Context(), severity, body, attrs, traceID, spanID); err != nil {
log.Printf("failed to emit sourcemap diagnostic: %v", err)
}
}
logResolve(plog.SeverityNumberInfo, "sourcemap resolution received", map[string]any{
"event.name": "sourcemap.resolve",
"http.route": "/api/sourcemaps/resolve",
"http.request.method": r.Method,
"sourcemap.resolver_ready": c.sourceMaps != nil,
})
var req struct {
Items []struct {
Release string `json:"release"`
@@ -525,11 +624,17 @@ func (c *Collector) HandleResolveStacks(w http.ResponseWriter, r *http.Request)
} `json:"items"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&req); err != nil {
logResolve(plog.SeverityNumberError, "sourcemap resolution failed: invalid body", map[string]any{
"event.name": "sourcemap.resolve.failed",
"sourcemap.failure_stage": "decode",
"error.type": "invalid_json",
})
http.Error(w, "invalid body", http.StatusBadRequest)
return
}
out := make([]string, len(req.Items))
frames := make([][]sourcemaps.Frame, len(req.Items))
statusCounts := map[string]int{}
for i, it := range req.Items {
out[i] = it.Stacktrace
frames[i] = []sourcemaps.Frame{}
@@ -537,7 +642,20 @@ func (c *Collector) HandleResolveStacks(w http.ResponseWriter, r *http.Request)
out[i] = c.sourceMaps.ResolveStack(it.Release, it.Stacktrace)
frames[i] = c.sourceMaps.ResolveFrames(it.Release, it.Stacktrace)
}
for _, frame := range frames[i] {
statusCounts[frame.Status]++
}
}
logResolve(plog.SeverityNumberInfo, "sourcemap resolution completed", map[string]any{
"event.name": "sourcemap.resolve.completed",
"sourcemap.item_count": len(req.Items),
"sourcemap.frame_count": statusCounts["mapped"] + statusCounts["no-release"] + statusCounts["no-map"] + statusCounts["unmapped"],
"sourcemap.mapped": statusCounts["mapped"],
"sourcemap.no_release": statusCounts["no-release"],
"sourcemap.no_map": statusCounts["no-map"],
"sourcemap.unmapped": statusCounts["unmapped"],
"sourcemap.resolver_ready": c.sourceMaps != nil,
})
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"resolved": out, "frames": frames})
}
+22
View File
@@ -175,6 +175,28 @@ func (c *Collector) HandleIssueTraces(w http.ResponseWriter, r *http.Request) {
}
}
// HandleIssueSources returns all distinct source IPs recorded for an issue.
// This is separate from /traces because trace pages intentionally bound the
// number of raw samples returned per trace.
func (c *Collector) HandleIssueSources(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
if c.occStore == nil {
http.Error(w, "issue source history requires ClickHouse storage", http.StatusNotImplemented)
return
}
sources, err := c.occStore.IssueSources(r.Context(), id)
if err != nil {
log.Printf("failed to load issue sources %s: %v", id, err)
http.Error(w, "failed to load issue sources", http.StatusInternalServerError)
return
}
if sources == nil {
sources = []storage.IssueSource{}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(sources)
}
// groupInMemory builds the same shape from the tracker's bounded samples,
// newest trace first.
func groupInMemory(samples []issues.Sample) ([]issueTraceJSON, []issues.Sample) {
+4
View File
@@ -34,6 +34,10 @@ func (o *occStore) IssueTraces(context.Context, string, storage.IssueTracesOptio
return o.page, nil
}
func (o *occStore) IssueSources(context.Context, string) ([]storage.IssueSource, error) {
return nil, nil
}
func (o *occStore) TraceEntries(context.Context, string, int) ([]websocket.HistoryEntry, error) {
return nil, nil
}
+110
View File
@@ -0,0 +1,110 @@
package collector
import (
"encoding/json"
"log"
"net/http"
"net/netip"
"strings"
"time"
"git.tornberg.me/go-otel/internal/features"
"git.tornberg.me/go-otel/internal/storage"
"git.tornberg.me/go-otel/internal/triage"
"git.tornberg.me/go-otel/internal/uaclass"
)
// SetSourceIPQuerier enables the source address endpoints.
func (c *Collector) SetSourceIPQuerier(q storage.SourceIPQuerier) { c.sourceIPs = q }
func (c *Collector) sourceIPsEnabled(w http.ResponseWriter) bool {
if c.sourceIPs == nil {
statsNotEnabled(w)
return false
}
return true
}
// HandleSourceIPs lists every unique source address ever recorded.
// Query: q (substring), hours (only addresses seen within), limit.
func (c *Collector) HandleSourceIPs(w http.ResponseWriter, r *http.Request) {
if !c.sourceIPsEnabled(w) {
return
}
var since time.Time
if _, hours := parseStatsCommon(r); r.URL.Query().Get("hours") != "" {
since = time.Now().Add(-time.Duration(hours) * time.Hour)
}
ips, err := c.sourceIPs.SourceIPs(r.Context(), strings.TrimSpace(r.URL.Query().Get("q")), since, parseLimit(r, 200))
if err != nil {
log.Printf("failed to list source ips: %v", err)
http.Error(w, "failed to list source ips", http.StatusInternalServerError)
return
}
// Judge scanning behaviour over the last week for the whole page in one
// query. A failure only drops the hints; the list itself is still useful.
addrs := make([]string, len(ips))
for i := range ips {
addrs[i] = ips[i].IP
}
if routes, err := c.sourceIPs.SourceIPRoutes(r.Context(), addrs, time.Now().Add(-7*24*time.Hour)); err != nil {
log.Printf("failed to assess source ips: %v", err)
} else {
for i := range ips {
ips[i].BadActor, ips[i].ProbePaths = assessSourceIP(ips[i].IP, routes[ips[i].IP])
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"ips": ips})
}
// HandleSourceIPActivity lists the recent requests made from one address.
// Query: hours (default 168), limit.
func (c *Collector) HandleSourceIPActivity(w http.ResponseWriter, r *http.Request) {
if !c.sourceIPsEnabled(w) {
return
}
addr, err := netip.ParseAddr(r.PathValue("ip"))
if err != nil {
http.Error(w, "invalid ip", http.StatusBadRequest)
return
}
_, hours := parseStatsCommon(r)
if r.URL.Query().Get("hours") == "" {
hours = 24 * 7
}
events, err := c.sourceIPs.SourceIPActivity(r.Context(), addr.String(), time.Now().Add(-time.Duration(hours)*time.Hour), parseLimit(r, 200))
if err != nil {
log.Printf("failed to load source ip activity: %v", err)
http.Error(w, "failed to load source ip activity", http.StatusInternalServerError)
return
}
reason, probes := assessSourceIP(addr.String(), events)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"ip": addr.String(), "hours": hours, "events": events,
// badActor is the first matching features.MarkBadActors rule over the
// returned window; probePaths is how many distinct probe paths it hit.
"badActor": reason, "probePaths": probes,
})
}
// assessSourceIP applies the same bad-actor rules the detector uses to one
// address's requests, so the hint in the UI never disagrees with triage.
func assessSourceIP(ip string, events []storage.SourceIPEvent) (reason string, probePaths int) {
fe := make([]features.Event, len(events))
probes := map[string]struct{}{}
for i, e := range events {
fe[i] = features.Event{Route: e.Route, StatusCode: int(e.StatusCode), ClientIP: ip, UAClass: uaclass.Of(e.UserAgent)}
if triage.IsProbePath(e.Route) {
probes[e.Route] = struct{}{}
}
}
features.MarkBadActors(fe, features.BadActorConfig{})
for _, e := range fe {
if e.BadActor != "" {
return e.BadActor, len(probes)
}
}
return "", len(probes)
}
+30
View File
@@ -0,0 +1,30 @@
package collector
import (
"fmt"
"testing"
"git.tornberg.me/go-otel/internal/storage"
)
func TestAssessSourceIP(t *testing.T) {
ev := func(route string, status uint16, ua string) storage.SourceIPEvent {
return storage.SourceIPEvent{Route: route, StatusCode: status, UserAgent: ua}
}
browser := "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120 Safari/537.36"
if r, _ := assessSourceIP("1.2.3.4", []storage.SourceIPEvent{ev("/", 200, browser), ev("/cart", 200, browser)}); r != "" {
t.Errorf("normal traffic flagged as %q", r)
}
probes := []storage.SourceIPEvent{ev("/.env", 404, browser), ev("/.git/config", 404, browser), ev("/wp-admin", 404, browser)}
if r, n := assessSourceIP("1.2.3.4", probes); r != "probe-scanner" || n != 3 {
t.Errorf("probes: got %q, %d", r, n)
}
var walk []storage.SourceIPEvent
for i := 0; i < 15; i++ {
walk = append(walk, ev(fmt.Sprintf("/missing-%d", i), 404, browser))
}
if r, _ := assessSourceIP("1.2.3.4", walk); r != "enumerator" {
t.Errorf("enumeration: got %q", r)
}
}