better

internal/collector/collector.go

@@ -5,9 +5,11 @@ import (
 	"context"
 	"encoding/json"
 	"log"
+	"strings"
 	"sync"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 
 	"go.opentelemetry.io/collector/consumer"
@@ -21,13 +23,18 @@ type Collector struct {
 	historySize  int
 	history      *ring.Ring
 	historyMutex sync.RWMutex
+	store        storage.Store
+	subscribers  map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool
+	subMutex     sync.Mutex
 }
 
-func New(broadcaster websocket.Broadcaster, historySize int) *Collector {
+func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store) *Collector {
 	return &Collector{
 		broadcaster: broadcaster,
 		historySize: historySize,
 		history:     ring.New(historySize),
+		store:       store,
+		subscribers: make(map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool),
 	}
 }
 
@@ -36,15 +43,59 @@ func (c *Collector) SetBroadcaster(broadcaster websocket.Broadcaster) {
 }
 
 func (c *Collector) addToHistory(dataType websocket.DataType, data interface{}) {
-	c.historyMutex.Lock()
-	defer c.historyMutex.Unlock()
-
-	c.history.Value = websocket.HistoryEntry{
+	entry := websocket.HistoryEntry{
 		Type:      string(dataType),
 		Timestamp: time.Now().UnixNano(),
 		Data:      data,
 	}
+
+	c.historyMutex.Lock()
+	c.history.Value = entry
 	c.history = c.history.Next()
+	c.historyMutex.Unlock()
+
+	if c.store != nil {
+		// Asynchronously append to store to avoid blocking ingestion
+		go func() {
+			if err := c.store.Append(context.Background(), entry); err != nil {
+				log.Printf("failed to append to store: %v", err)
+			}
+		}()
+	}
+
+	c.notifySubscribers(entry)
+}
+
+func (c *Collector) notifySubscribers(entry websocket.HistoryEntry) {
+	c.subMutex.Lock()
+	defer c.subMutex.Unlock()
+
+	for ch, filter := range c.subscribers {
+		if filter(entry) {
+			select {
+			case ch <- entry:
+				// Successfully sent
+			default:
+				// Channel blocked, skip
+			}
+		}
+	}
+}
+
+func (c *Collector) Subscribe(filter func(websocket.HistoryEntry) bool) (chan websocket.HistoryEntry, func()) {
+	ch := make(chan websocket.HistoryEntry, 1)
+	c.subMutex.Lock()
+	c.subscribers[ch] = filter
+	c.subMutex.Unlock()
+
+	cleanup := func() {
+		c.subMutex.Lock()
+		delete(c.subscribers, ch)
+		c.subMutex.Unlock()
+		close(ch)
+	}
+
+	return ch, cleanup
 }
 
 func (c *Collector) GetHistory() []websocket.HistoryEntry {
@@ -60,6 +111,47 @@ func (c *Collector) GetHistory() []websocket.HistoryEntry {
 	return history
 }
 
+func (c *Collector) SearchHistory(ctx context.Context, query string, limit int) ([]websocket.HistoryEntry, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+
+	if c.store != nil {
+		return c.store.Search(ctx, query, limit)
+	}
+
+	// Fallback to in-memory if no store
+	history := c.GetHistory()
+	var filtered []websocket.HistoryEntry
+	// Lowercase query for case-insensitive search
+	lowerQuery := strings.ToLower(query)
+
+	for _, entry := range history {
+		if query == "" {
+			filtered = append(filtered, entry)
+		} else {
+			// Basic string filter for in-memory
+			data, _ := json.Marshal(entry.Data)
+			if strings.Contains(strings.ToLower(string(data)), lowerQuery) {
+				filtered = append(filtered, entry)
+			}
+		}
+		// Keep it simple and just filter all then limit, since historySize is usually small
+	}
+
+	// Respect limit and return latest messages first
+	if len(filtered) > limit {
+		filtered = filtered[len(filtered)-limit:]
+	}
+
+	// Reverse to match Store's Search behavior (latest first)
+	for i, j := 0, len(filtered)-1; i < j; i, j = i+1, j-1 {
+		filtered[i], filtered[j] = filtered[j], filtered[i]
+	}
+
+	return filtered, nil
+}
+
 func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
 	marshaler := &ptrace.JSONMarshaler{}
 	buf, err := marshaler.MarshalTraces(td)

internal/collector/collector_test.go

@@ -0,0 +1,52 @@
+package collector
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+type mockStore struct {
+	entries []websocket.HistoryEntry
+}
+
+func (m *mockStore) Append(ctx context.Context, entry websocket.HistoryEntry) error {
+	m.entries = append(m.entries, entry)
+	return nil
+}
+
+func (m *mockStore) GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error) {
+	return m.entries, nil
+}
+
+func (m *mockStore) Search(ctx context.Context, query string, limit int) ([]websocket.HistoryEntry, error) {
+	return m.entries, nil
+}
+
+func TestCollectorWithStore(t *testing.T) {
+	store := &mockStore{}
+	coll := New(nil, 10, store)
+
+	coll.addToHistory(websocket.DataTypeLogs, map[string]interface{}{"msg": "test"})
+
+	// Give it a moment since it's asynchronous
+	timeout := time.After(100 * time.Millisecond)
+	for {
+		select {
+		case <-timeout:
+			t.Fatal("timed out waiting for entry to be appended to store")
+		default:
+			if len(store.entries) > 0 {
+				goto OK
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+OK:
+	if store.entries[0].Type != "logs" {
+		t.Errorf("expected type 'logs', got %s", store.entries[0].Type)
+	}
+}

internal/storage/file_store.go

@@ -0,0 +1,83 @@
+package storage
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+	"github.com/natefinch/lumberjack"
+)
+
+type FileStore struct {
+	logger *lumberjack.Logger
+}
+
+func NewFileStore(filename string) *FileStore {
+	return &FileStore{
+		logger: &lumberjack.Logger{
+			Filename:   filename,
+			MaxSize:    100, // megabytes
+			MaxBackups: 3,
+			MaxAge:     28,   // days
+			Compress:   true, // disabled by default
+		},
+	}
+}
+
+func (fs *FileStore) Append(ctx context.Context, entry websocket.HistoryEntry) error {
+	data, err := json.Marshal(entry)
+	if err != nil {
+		return err
+	}
+	_, err = fs.logger.Write(append(data, '\n'))
+	return err
+}
+
+func (fs *FileStore) GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error) {
+	return fs.Search(ctx, "", limit)
+}
+
+func (fs *FileStore) Search(ctx context.Context, query string, limit int) ([]websocket.HistoryEntry, error) {
+	if limit <= 0 {
+		limit = 50 // Default to a manageable number
+	}
+
+	file, err := os.Open(fs.logger.Filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil // No logs yet
+		}
+		return nil, err
+	}
+	defer file.Close()
+
+	var matchingLines []string
+	scanner := bufio.NewScanner(file)
+	// Lowercase query for case-insensitive search
+	lowerQuery := strings.ToLower(query)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if query == "" || strings.Contains(strings.ToLower(line), lowerQuery) {
+			matchingLines = append(matchingLines, line)
+			// Only keep the most recent 'limit' matches to save memory
+			if len(matchingLines) > limit {
+				matchingLines = matchingLines[1:]
+			}
+		}
+	}
+
+	var results []websocket.HistoryEntry
+	// Reverse to get latest first (since matchingLines contains them in chronological order)
+	for i := len(matchingLines) - 1; i >= 0; i-- {
+		var entry websocket.HistoryEntry
+		if err := json.Unmarshal([]byte(matchingLines[i]), &entry); err == nil {
+			results = append(results, entry)
+		}
+	}
+
+	return results, scanner.Err()
+}

internal/storage/file_store_test.go

@@ -0,0 +1,64 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+func TestFileStore(t *testing.T) {
+	filename := "test-otel.log"
+	defer os.Remove(filename)
+
+	fs := NewFileStore(filename)
+	ctx := context.Background()
+
+	entry1 := websocket.HistoryEntry{
+		Type:      "logs",
+		Timestamp: 100,
+		Data:      map[string]interface{}{"msg": "hello world"},
+	}
+	entry2 := websocket.HistoryEntry{
+		Type:      "traces",
+		Timestamp: 200,
+		Data:      map[string]interface{}{"msg": "trace event"},
+	}
+
+	if err := fs.Append(ctx, entry1); err != nil {
+		t.Fatalf("failed to append entry1: %v", err)
+	}
+	if err := fs.Append(ctx, entry2); err != nil {
+		t.Fatalf("failed to append entry2: %v", err)
+	}
+
+	// Test GetHistory (limit 1)
+	history, err := fs.GetHistory(ctx, 1)
+	if err != nil {
+		t.Fatalf("GetHistory failed: %v", err)
+	}
+	if len(history) != 1 {
+		t.Errorf("expected 1 entry, got %d", len(history))
+	}
+	// Should be latest entry
+	if history[0].Type != "traces" {
+		t.Errorf("expected latest entry type 'traces', got %s", history[0].Type)
+	}
+
+	// Test Search
+	results, err := fs.Search(ctx, "hello", 10)
+	if err != nil {
+		t.Fatalf("Search failed: %v", err)
+	}
+	if len(results) != 1 {
+		t.Errorf("expected 1 result for search 'hello', got %d", len(results))
+	}
+
+    // Check content of result
+    data, _ := json.Marshal(results[0].Data)
+    if string(data) != `{"msg":"hello world"}` {
+        t.Errorf("unexpected data content: %s", string(data))
+    }
+}

internal/storage/store.go

@@ -0,0 +1,12 @@
+package storage
+
+import (
+	"context"
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+type Store interface {
+	Append(ctx context.Context, entry websocket.HistoryEntry) error
+	GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error)
+	Search(ctx context.Context, query string, limit int) ([]websocket.HistoryEntry, error)
+}