diff --git a/build.sh b/build.sh
index 72459fd..38c3514 100755
--- a/build.sh
+++ b/build.sh
@@ -17,22 +17,9 @@ docker build \
docker push matst80/otel-debug:latest
-# Shared key that only keeps strangers from uploading source maps (it is not a
-# real credential). Taken from the environment, else from the cluster secret,
-# else generated and stored there so the deployment can mount it.
-ensure_sourcemap_key() {
- if [[ -z "${OTEL_SOURCEMAP_KEY:-}" ]]; then
- OTEL_SOURCEMAP_KEY="$(kubectl get secret otel-sourcemap-key -n monitoring \
- -o jsonpath='{.data.key}' 2>/dev/null | base64 -d || true)"
- fi
- if [[ -z "${OTEL_SOURCEMAP_KEY}" ]]; then
- OTEL_SOURCEMAP_KEY="$(openssl rand -hex 24)"
- fi
- kubectl create secret generic otel-sourcemap-key -n monitoring \
- --from-literal=key="${OTEL_SOURCEMAP_KEY}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
- export OTEL_SOURCEMAP_KEY
-}
-ensure_sourcemap_key
+# Shared key that only keeps strangers from uploading source maps. It is not a
+# secret, so it is hardcoded here and in k8s/deployment.yaml.
+export OTEL_SOURCEMAP_KEY="6901e62e4468631f27f1dc85fd4231f35b5e591c61f5bcfa"
# Roll out first: the new image requires OTEL_SOURCEMAP_KEY, so the manifest has
# to be applied before uploads, or any pod restart in between rejects them.
diff --git a/cmd/server/main.go b/cmd/server/main.go
index a0e9f7c..cffb4f6 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -22,6 +22,7 @@ import (
"git.tornberg.me/go-otel/internal/prometheus"
"git.tornberg.me/go-otel/internal/sourcemaps"
"git.tornberg.me/go-otel/internal/storage"
+ "git.tornberg.me/go-otel/internal/todos"
"git.tornberg.me/go-otel/internal/topology"
"git.tornberg.me/go-otel/internal/websocket"
"go.opentelemetry.io/collector/pdata/plog"
@@ -285,6 +286,28 @@ func main() {
investigationStore = st
}
}
+ var todoStore todos.Store
+ if chStore, ok := store.(*storage.ClickHouseStore); ok {
+ if st, err := todos.NewClickHouseStore(context.Background(), chStore.Conn(), chStore.Database()); err != nil {
+ log.Printf("todos: clickhouse unavailable, using file store: %v", err)
+ } else {
+ todoStore = st
+ }
+ }
+ if todoStore == nil {
+ path := os.Getenv("OTEL_TODOS_PATH")
+ if path == "" {
+ path = "otel-todos.jsonl"
+ }
+ if st, err := todos.NewFileStore(path); err != nil {
+ log.Printf("todos disabled: %v", err)
+ } else {
+ todoStore = st
+ }
+ }
+ if todoStore != nil {
+ coll.SetTodoStore(todoStore)
+ }
investigator := investigation.NewHandler(investigation.ConfigFromEnv(), investigationStore)
abuse := abuseipdb.NewHandler(abuseipdb.ConfigFromEnv())
@@ -324,6 +347,7 @@ func main() {
otelMux.HandleFunc("/v1/logs", withCORS(coll.HandleLogs))
otelMux.HandleFunc("/v1/logs/", withCORS(coll.HandleLogs))
otelMux.HandleFunc("/v1/vitals", withCORS(coll.HandleVitals))
+ otelMux.HandleFunc("/v1/todos", withCORS(coll.HandleTodoCreate))
otelMux.HandleFunc("/health", coll.HandleHealth)
// UI and API mux (port 8080)
@@ -338,6 +362,7 @@ func main() {
uiMux.HandleFunc("/v1/logs", withCORS(coll.HandleLogs))
uiMux.HandleFunc("/v1/logs/", withCORS(coll.HandleLogs))
uiMux.HandleFunc("/v1/vitals", withCORS(coll.HandleVitals))
+ uiMux.HandleFunc("/v1/todos", withCORS(coll.HandleTodoCreate))
uiMux.HandleFunc("/health", coll.HandleHealth)
uiMux.HandleFunc("GET /internal/honeypot/ips", honeypotTracker.HandleIPs)
@@ -383,6 +408,7 @@ func main() {
uiMux.Handle("PATCH /api/issues/{id}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueByID)))
uiMux.Handle("GET /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueSimilar)))
uiMux.Handle("POST /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleSimilarityFeedback)))
+ uiMux.Handle("GET /api/todos", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleTodoList)))
uiMux.Handle("GET /api/patterns", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandlePatterns)))
uiMux.Handle("GET /api/topology", oidcAuth.RequireAuth(http.HandlerFunc(topo.HandleTopology)))
uiMux.Handle("GET /api/rollouts", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleRollouts)))
diff --git a/instrumentation.md b/instrumentation.md
index 0d46ccf..235685a 100644
--- a/instrumentation.md
+++ b/instrumentation.md
@@ -89,6 +89,34 @@ export class AppErrorBoundary extends React.Component<
Use the same pattern in your application, including a user-visible fallback
for individual views and the application root.
+## Todo widget
+
+`otel-todo.js` adds a small floating symbol and a keyboard shortcut
+(default `Alt+Shift+T`) that opens a corner dialog for filing a todo/issue
+from the page, with an optional component picker. The current path and hash
+are posted with it.
+
+`otel-rum.js` loads it automatically (from the same directory as the RUM
+script) and reuses its service name, base URL and ingest key. Opt out with
+`OtelRUM.init({ excludeTodo: true })` or `data-exclude-todo="true"`. Or load it
+standalone:
+
+```html
+
+```
+
+Components are also discovered from `[data-component="..."]` elements. Other
+options (on the RUM or todo script, or `OtelRUM.init({ todo: {...} })`): `data-shortcut`, `data-hide-button="true"`, `data-base-url`.
+
+- `POST /v1/todos` (ingest key, CORS) body `{project, text, component, path, hash, url, ua, session}`
+- `GET /api/todos?project=&limit=100` (authenticated; omit `project` for all), newest first
+
+Todos go to ClickHouse when configured, else to the JSONL file at `OTEL_TODOS_PATH`
+(default `otel-todos.jsonl`).
+
## Backend instrumentation
Instrument backend services with an OpenTelemetry SDK or auto-instrumentation
diff --git a/internal/collector/collector.go b/internal/collector/collector.go
index 3fdc8f5..ba2d043 100644
--- a/internal/collector/collector.go
+++ b/internal/collector/collector.go
@@ -18,6 +18,7 @@ import (
"git.tornberg.me/go-otel/internal/prometheus"
"git.tornberg.me/go-otel/internal/sourcemaps"
"git.tornberg.me/go-otel/internal/storage"
+ "git.tornberg.me/go-otel/internal/todos"
"git.tornberg.me/go-otel/internal/uaclass"
"git.tornberg.me/go-otel/internal/websocket"
@@ -43,6 +44,7 @@ type Collector struct {
ingestLimiter *ipRateLimiter
ingestKey string
+ todoStore todos.Store
sourceMapKey string
notifyIssuesFn func()
diff --git a/internal/collector/todos.go b/internal/collector/todos.go
new file mode 100644
index 0000000..2013b3c
--- /dev/null
+++ b/internal/collector/todos.go
@@ -0,0 +1,118 @@
+package collector
+
+import (
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "log"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "git.tornberg.me/go-otel/internal/todos"
+)
+
+const (
+ maxTodoBody = 16 << 10
+ maxTodoText = 4000
+ maxTodoMeta = 500
+)
+
+// SetTodoStore enables the todo endpoints.
+func (c *Collector) SetTodoStore(s todos.Store) { c.todoStore = s }
+
+func clipTo(s string, n int) string {
+ s = strings.TrimSpace(s)
+ if len(s) > n {
+ return s[:n]
+ }
+ return s
+}
+
+type todoPayload struct {
+ Project string `json:"project"`
+ Text string `json:"text"`
+ Component string `json:"component"`
+ Path string `json:"path"`
+ Hash string `json:"hash"`
+ URL string `json:"url"`
+ UserAgent string `json:"ua"`
+ Session string `json:"session"`
+}
+
+// HandleTodoCreate ingests a todo posted by otel-todo.js. It is guarded by
+// the same ingest key and rate limit as the other RUM endpoints.
+func (c *Collector) HandleTodoCreate(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if c.todoStore == nil {
+ http.Error(w, "todos not enabled", http.StatusServiceUnavailable)
+ return
+ }
+ if !c.checkIngest(w, r) {
+ return
+ }
+ var p todoPayload
+ if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxTodoBody)).Decode(&p); err != nil {
+ http.Error(w, "invalid todo payload", http.StatusBadRequest)
+ return
+ }
+ text := clipTo(p.Text, maxTodoText)
+ if text == "" {
+ http.Error(w, "text is required", http.StatusBadRequest)
+ return
+ }
+ project := clipTo(p.Project, maxTodoMeta)
+ if project == "" {
+ project = "frontend"
+ }
+ id := make([]byte, 8)
+ _, _ = rand.Read(id)
+ t := todos.Todo{
+ ID: hex.EncodeToString(id),
+ Project: project,
+ Text: text,
+ Component: clipTo(p.Component, maxTodoMeta),
+ Path: clipTo(p.Path, maxTodoMeta),
+ Hash: clipTo(p.Hash, maxTodoMeta),
+ URL: clipTo(p.URL, maxTodoMeta),
+ UserAgent: clipTo(p.UserAgent, maxTodoMeta),
+ Session: clipTo(p.Session, maxTodoMeta),
+ CreatedAt: time.Now().UTC(),
+ }
+ if err := c.todoStore.Add(r.Context(), t); err != nil {
+ log.Printf("failed to store todo: %v", err)
+ http.Error(w, "failed to store todo", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusCreated)
+ _ = json.NewEncoder(w).Encode(t)
+}
+
+// HandleTodoList returns todos, newest first. ?project= filters by project
+// (omit for all); ?limit= defaults to 100, max 1000. Registered behind auth.
+func (c *Collector) HandleTodoList(w http.ResponseWriter, r *http.Request) {
+ if c.todoStore == nil {
+ http.Error(w, "todos not enabled", http.StatusServiceUnavailable)
+ return
+ }
+ limit := 100
+ if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && v > 0 {
+ limit = min(v, 1000)
+ }
+ items, err := c.todoStore.List(r.Context(), r.URL.Query().Get("project"), limit)
+ if err != nil {
+ log.Printf("failed to list todos: %v", err)
+ http.Error(w, "failed to list todos", http.StatusInternalServerError)
+ return
+ }
+ if items == nil {
+ items = []todos.Todo{}
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(items)
+}
diff --git a/internal/collector/todos_test.go b/internal/collector/todos_test.go
new file mode 100644
index 0000000..d815a53
--- /dev/null
+++ b/internal/collector/todos_test.go
@@ -0,0 +1,45 @@
+package collector
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "git.tornberg.me/go-otel/internal/todos"
+)
+
+func TestTodoCreateAndList(t *testing.T) {
+ coll := New(nil, 10, nil)
+ st, err := todos.NewFileStore(filepath.Join(t.TempDir(), "todos.jsonl"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ coll.SetTodoStore(st)
+
+ post := func(body string) *httptest.ResponseRecorder {
+ w := httptest.NewRecorder()
+ coll.HandleTodoCreate(w, httptest.NewRequest(http.MethodPost, "/v1/todos", strings.NewReader(body)))
+ return w
+ }
+ if w := post(`{"project":"a","text":"fix cart","component":"Cart","path":"/cart","hash":"#x"}`); w.Code != http.StatusCreated {
+ t.Fatalf("create: %d %s", w.Code, w.Body)
+ }
+ post(`{"project":"b","text":"other"}`)
+ if w := post(`{"project":"a","text":" "}`); w.Code != http.StatusBadRequest {
+ t.Fatalf("empty text: %d", w.Code)
+ }
+
+ list := func(q string) string {
+ w := httptest.NewRecorder()
+ coll.HandleTodoList(w, httptest.NewRequest(http.MethodGet, "/api/todos"+q, nil))
+ return w.Body.String()
+ }
+ if got := list("?project=a"); !strings.Contains(got, "fix cart") || strings.Contains(got, "other") || !strings.Contains(got, `"hash":"#x"`) {
+ t.Fatalf("project filter: %s", got)
+ }
+ if got := list(""); !strings.Contains(got, "fix cart") || !strings.Contains(got, "other") {
+ t.Fatalf("all: %s", got)
+ }
+}
diff --git a/internal/todos/todos.go b/internal/todos/todos.go
new file mode 100644
index 0000000..8fa72d0
--- /dev/null
+++ b/internal/todos/todos.go
@@ -0,0 +1,155 @@
+// Package todos stores todo/issue notes filed from the browser by the
+// otel-todo.js widget.
+package todos
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+// Todo is one note filed from a page. Project is the RUM service name.
+type Todo struct {
+ ID string `json:"id"`
+ Project string `json:"project"`
+ Text string `json:"text"`
+ Component string `json:"component,omitempty"`
+ Path string `json:"path,omitempty"`
+ Hash string `json:"hash,omitempty"`
+ URL string `json:"url,omitempty"`
+ UserAgent string `json:"userAgent,omitempty"`
+ Session string `json:"session,omitempty"`
+ CreatedAt time.Time `json:"createdAt"`
+}
+
+// Store persists todos. List returns newest first; an empty project means all.
+type Store interface {
+ Add(ctx context.Context, t Todo) error
+ List(ctx context.Context, project string, limit int) ([]Todo, error)
+}
+
+// FileStore keeps todos in memory and appends them to a JSONL file.
+type FileStore struct {
+ mu sync.Mutex
+ path string
+ items []Todo
+}
+
+func NewFileStore(path string) (*FileStore, error) {
+ fs := &FileStore{path: path}
+ f, err := os.Open(path)
+ if os.IsNotExist(err) {
+ return fs, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ sc := bufio.NewScanner(f)
+ sc.Buffer(make([]byte, 64<<10), 1<<20)
+ for sc.Scan() {
+ var t Todo
+ if json.Unmarshal(sc.Bytes(), &t) == nil {
+ fs.items = append(fs.items, t)
+ }
+ }
+ return fs, sc.Err()
+}
+
+func (s *FileStore) Add(_ context.Context, t Todo) error {
+ line, err := json.Marshal(t)
+ if err != nil {
+ return err
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ f, err := os.OpenFile(s.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if _, err := f.Write(append(line, '\n')); err != nil {
+ return err
+ }
+ s.items = append(s.items, t)
+ return nil
+}
+
+func (s *FileStore) List(_ context.Context, project string, limit int) ([]Todo, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ var out []Todo
+ for _, t := range s.items {
+ if project == "" || t.Project == project {
+ out = append(out, t)
+ }
+ }
+ sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
+ if limit > 0 && len(out) > limit {
+ out = out[:limit]
+ }
+ return out, nil
+}
+
+// ClickHouseStore persists todos in a plain MergeTree table (no TTL).
+type ClickHouseStore struct {
+ conn driver.Conn
+ db string
+}
+
+func NewClickHouseStore(ctx context.Context, conn driver.Conn, db string) (*ClickHouseStore, error) {
+ err := conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.todos (
+ id String,
+ project LowCardinality(String),
+ text String,
+ component String,
+ path String,
+ hash String,
+ url String,
+ user_agent String,
+ session String,
+ created_at DateTime64(3, 'UTC')
+ ) ENGINE = MergeTree
+ ORDER BY (project, created_at)`, db))
+ if err != nil {
+ return nil, err
+ }
+ return &ClickHouseStore{conn: conn, db: db}, nil
+}
+
+func (s *ClickHouseStore) Add(ctx context.Context, t Todo) error {
+ return s.conn.Exec(ctx, fmt.Sprintf(`INSERT INTO %s.todos (id, project, text, component, path, hash, url, user_agent, session, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, s.db),
+ t.ID, t.Project, t.Text, t.Component, t.Path, t.Hash, t.URL, t.UserAgent, t.Session, t.CreatedAt.UTC())
+}
+
+func (s *ClickHouseStore) List(ctx context.Context, project string, limit int) ([]Todo, error) {
+ q := fmt.Sprintf(`SELECT id, project, text, component, path, hash, url, user_agent, session, created_at FROM %s.todos`, s.db)
+ var args []any
+ if project != "" {
+ q += ` WHERE project = ?`
+ args = append(args, project)
+ }
+ q += ` ORDER BY created_at DESC LIMIT ?`
+ args = append(args, limit)
+ rows, err := s.conn.Query(ctx, q, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ var out []Todo
+ for rows.Next() {
+ var t Todo
+ if err := rows.Scan(&t.ID, &t.Project, &t.Text, &t.Component, &t.Path, &t.Hash, &t.URL, &t.UserAgent, &t.Session, &t.CreatedAt); err != nil {
+ return nil, err
+ }
+ out = append(out, t)
+ }
+ return out, rows.Err()
+}
diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index c9b2fa0..091e33c 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -52,10 +52,7 @@ spec:
# Shared key required to POST /api/sourcemaps (X-Sourcemap-Key header);
# uploads are refused while it is unset.
- name: OTEL_SOURCEMAP_KEY
- valueFrom:
- secretKeyRef:
- name: otel-sourcemap-key
- key: key
+ value: "6901e62e4468631f27f1dc85fd4231f35b5e591c61f5bcfa"
# AbuseIPDB API key for authenticated IP checks/reports from the UI.
# Create out of band: kubectl create secret generic abuseipdb-key -n monitoring --from-literal=key=...
- name: ABUSE_KEY
diff --git a/scripts/upload-sourcemap.sh b/scripts/upload-sourcemap.sh
index af5b753..accb472 100755
--- a/scripts/upload-sourcemap.sh
+++ b/scripts/upload-sourcemap.sh
@@ -2,7 +2,7 @@
set -euo pipefail
: "${OTEL_URL:?set OTEL_URL, for example https://otel.example.com}"
-: "${OTEL_SOURCEMAP_KEY:?set OTEL_SOURCEMAP_KEY to the shared upload key}"
+OTEL_SOURCEMAP_KEY="${OTEL_SOURCEMAP_KEY:-6901e62e4468631f27f1dc85fd4231f35b5e591c61f5bcfa}"
: "${RELEASE:?set RELEASE to the exact VITE_APP_RELEASE used at build time}"
bundle="${1:?usage: RELEASE=... OTEL_URL=... $0 dist/assets/index-HASH.js}"
diff --git a/static/otel-rum.js b/static/otel-rum.js
index d143b03..e88132d 100644
--- a/static/otel-rum.js
+++ b/static/otel-rum.js
@@ -142,6 +142,13 @@
var trackSpaRoutes = config.trackSpaRoutes !== false;
var reportHttpFailures = config.reportHttpFailures !== false;
var captureWebVitals = config.captureWebVitals !== false;
+ // The todo widget (otel-todo.js) is loaded automatically unless excluded:
+ // init({ excludeTodo: true }) or data-exclude-todo="true".
+ var excludeTodo = config.excludeTodo === true || (scriptEl && scriptEl.getAttribute('data-exclude-todo') === 'true');
+ var todoLibUrl =
+ config.todoLibUrl ||
+ (scriptEl && scriptEl.getAttribute('data-todo-lib')) ||
+ (scriptEl && scriptEl.src ? scriptEl.src.replace(/[^\/]*$/, '') + 'otel-todo.js' : baseUrl + '/static/otel-todo.js');
var injectTraceHeaders = config.injectTraceHeaders !== false;
var traceHeaderOrigins = config.traceHeaderOrigins; // optional array; defaults to same-origin only
// Client-side rate limiting: max events per rolling minute.
@@ -406,6 +413,31 @@
global.document.head.appendChild(script);
}
+ function loadTodoWidget() {
+ var todo = config.todo || {};
+ function start() {
+ if (!global.OtelTodo) return;
+ global.OtelTodo.init({
+ baseUrl: baseUrl,
+ serviceName: serviceName,
+ ingestKey: ingestKey,
+ components: todo.components || ((scriptEl && scriptEl.getAttribute('data-components')) || '').split(','),
+ shortcut: todo.shortcut || (scriptEl && scriptEl.getAttribute('data-shortcut')) || '',
+ hideButton: todo.hideButton === true || (scriptEl && scriptEl.getAttribute('data-hide-button') === 'true')
+ });
+ }
+ if (global.OtelTodo) return start();
+ var s = global.document.createElement('script');
+ s.async = true;
+ s.src = todoLibUrl;
+ s.onload = start;
+ global.document.head.appendChild(s);
+ }
+ if (!excludeTodo && global.document) {
+ if (global.document.body) loadTodoWidget();
+ else global.document.addEventListener('DOMContentLoaded', loadTodoWidget);
+ }
+
// traceHeaders returns a headers object carrying a fresh (or
// continued) W3C traceparent, ready to spread into any HTTP client's
// request headers - fetch, XHR, axios, etc.
diff --git a/static/otel-todo.js b/static/otel-todo.js
new file mode 100644
index 0000000..f0f36bc
--- /dev/null
+++ b/static/otel-todo.js
@@ -0,0 +1,191 @@
+/**
+ * otel-todo.js - file todos / issues straight from the page.
+ *
+ * Shows a small floating symbol and listens for a keyboard shortcut. Either
+ * opens a native