fixes

internal/issues/statusstore.go

@@ -0,0 +1,122 @@
+package issues
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+// TriageState is the part of an issue a person decided, kept across restarts.
+// The tracker is in-memory, so without this every deploy would reopen every
+// resolved and ignored issue.
+type TriageState struct {
+	Status     string `json:"status"`
+	ResolvedAt int64  `json:"resolvedAt,omitempty"`
+}
+
+// StatusStore persists triage state by issue id.
+type StatusStore interface {
+	Load(ctx context.Context) (map[string]TriageState, error)
+	Save(ctx context.Context, id string, s TriageState) error
+}
+
+// FileStatusStore keeps all states in one JSON file, rewritten on each save.
+type FileStatusStore struct {
+	mu     sync.Mutex
+	path   string
+	states map[string]TriageState
+}
+
+func NewFileStatusStore(path string) (*FileStatusStore, error) {
+	fs := &FileStatusStore{path: path, states: map[string]TriageState{}}
+	b, err := os.ReadFile(path)
+	if os.IsNotExist(err) {
+		return fs, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &fs.states); err != nil {
+		return nil, err
+	}
+	return fs, nil
+}
+
+func (s *FileStatusStore) Load(context.Context) (map[string]TriageState, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := make(map[string]TriageState, len(s.states))
+	for k, v := range s.states {
+		out[k] = v
+	}
+	return out, nil
+}
+
+func (s *FileStatusStore) Save(_ context.Context, id string, st TriageState) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if st.Status == StatusOpen {
+		delete(s.states, id)
+	} else {
+		s.states[id] = st
+	}
+	b, err := json.Marshal(s.states)
+	if err != nil {
+		return err
+	}
+	tmp := s.path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, s.path)
+}
+
+// ClickHouseStatusStore keeps the latest state per issue id.
+type ClickHouseStatusStore struct {
+	conn driver.Conn
+	db   string
+}
+
+func NewClickHouseStatusStore(ctx context.Context, conn driver.Conn, db string) (*ClickHouseStatusStore, error) {
+	err := conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.issue_status (
+		id String,
+		status LowCardinality(String),
+		resolved_at Int64,
+		updated_at DateTime64(3, 'UTC')
+	) ENGINE = ReplacingMergeTree(updated_at)
+	ORDER BY id`, db))
+	if err != nil {
+		return nil, err
+	}
+	return &ClickHouseStatusStore{conn: conn, db: db}, nil
+}
+
+func (s *ClickHouseStatusStore) Load(ctx context.Context) (map[string]TriageState, error) {
+	rows, err := s.conn.Query(ctx, fmt.Sprintf(`SELECT id, status, resolved_at FROM %s.issue_status FINAL`, s.db))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]TriageState{}
+	for rows.Next() {
+		var id string
+		var st TriageState
+		if err := rows.Scan(&id, &st.Status, &st.ResolvedAt); err != nil {
+			return nil, err
+		}
+		if st.Status != StatusOpen {
+			out[id] = st
+		}
+	}
+	return out, rows.Err()
+}
+
+func (s *ClickHouseStatusStore) Save(ctx context.Context, id string, st TriageState) error {
+	return s.conn.Exec(ctx, fmt.Sprintf(`INSERT INTO %s.issue_status (id, status, resolved_at, updated_at) VALUES (?, ?, ?, ?)`, s.db),
+		id, st.Status, st.ResolvedAt, time.Now().UTC())
+}

internal/issues/statusstore_test.go

@@ -0,0 +1,50 @@
+package issues
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestTriagePersistsAcrossRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "status.json")
+	store, _ := NewFileStatusStore(path)
+	tr := NewTracker()
+	if err := tr.SetStatusStore(context.Background(), store); err != nil {
+		t.Fatal(err)
+	}
+	tr.Record("span", "svc", "boom", Sample{Timestamp: 1})
+	id := Fingerprint("span", "svc", NormalizeTitle("boom"))
+	if _, ok := tr.SetStatus(id, StatusIgnored); !ok {
+		t.Fatal("set status")
+	}
+	time.Sleep(100 * time.Millisecond) // async save
+
+	// "Restart": fresh tracker, same file.
+	store2, err := NewFileStatusStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr2 := NewTracker()
+	if err := tr2.SetStatusStore(context.Background(), store2); err != nil {
+		t.Fatal(err)
+	}
+	tr2.Record("span", "svc", "boom", Sample{Timestamp: 2})
+	got, _ := tr2.Get(id)
+	if got.Status != StatusIgnored {
+		t.Fatalf("status after restart = %q, want ignored", got.Status)
+	}
+}
+
+func TestResolvedIssueRegresses(t *testing.T) {
+	tr := NewTracker()
+	tr.Record("span", "svc", "boom", Sample{Timestamp: 1})
+	id := Fingerprint("span", "svc", NormalizeTitle("boom"))
+	tr.SetStatus(id, StatusResolved)
+	tr.Record("span", "svc", "boom", Sample{Timestamp: 5})
+	got, _ := tr.Get(id)
+	if got.Status != StatusOpen || got.RegressedAt != 5 {
+		t.Fatalf("got status=%q regressedAt=%d", got.Status, got.RegressedAt)
+	}
+}

internal/issues/tracker.go

@@ -3,8 +3,10 @@
 package issues
 
 import (
+	"context"
 	"fmt"
 	"hash/fnv"
+	"log"
 	"regexp"
 	"sort"
 	"strings"
@@ -86,6 +88,8 @@ type Issue struct {
 	// Status is the triage state: open, resolved or ignored.
 	Status     string `json:"status"`
 	ResolvedAt int64  `json:"resolvedAt,omitempty"`
+	// RegressedAt is when a resolved issue last fired again and was reopened.
+	RegressedAt int64 `json:"regressedAt,omitempty"`
 	// NormalizedTitle is the deterministic, redacted form used for exact grouping.
 	NormalizedTitle string `json:"normalizedTitle,omitempty"`
 	// PatternID groups related exact issues without merging their operational IDs.
@@ -117,10 +121,14 @@ type Pattern struct {
 // Tracker incrementally fingerprints and deduplicates errors as they're
 // recorded. It is safe for concurrent use.
 type Tracker struct {
-	mu       sync.Mutex
-	issues   map[string]*Issue
-	feedback map[string]string
-	sink     OccurrenceSink
+	mu     sync.Mutex
+	issues map[string]*Issue
+	// triage holds persisted resolved/ignored decisions by issue id, applied
+	// when an issue is (re)created after a restart.
+	triage      map[string]TriageState
+	statusStore StatusStore
+	feedback    map[string]string
+	sink        OccurrenceSink
 }
 
 // OccurrenceSink receives every recorded occurrence (already normalized) so
@@ -161,7 +169,43 @@ type ListPage struct {
 
 // NewTracker creates an empty Tracker.
 func NewTracker() *Tracker {
-	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)}
+	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string), triage: make(map[string]TriageState)}
+}
+
+// SetStatusStore loads persisted triage decisions and saves future ones.
+func (t *Tracker) SetStatusStore(ctx context.Context, store StatusStore) error {
+	states, err := store.Load(ctx)
+	if err != nil {
+		return err
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.statusStore = store
+	for id, st := range states {
+		t.triage[id] = st
+		if issue, ok := t.issues[id]; ok {
+			issue.Status, issue.ResolvedAt = st.Status, st.ResolvedAt
+		}
+	}
+	return nil
+}
+
+// persistLocked records a decision. The write happens off the lock.
+func (t *Tracker) persistLocked(id string, st TriageState) {
+	if st.Status == StatusOpen {
+		delete(t.triage, id)
+	} else {
+		t.triage[id] = st
+	}
+	if t.statusStore == nil {
+		return
+	}
+	store := t.statusStore
+	go func() {
+		if err := store.Save(context.Background(), id, st); err != nil {
+			log.Printf("failed to persist issue status: %v", err)
+		}
+	}()
 }
 
 // Record adds an occurrence of an error, creating a new issue or merging into
@@ -196,6 +240,9 @@ func (t *Tracker) record(kind, service, title string, sample Sample) (string, Oc
 			Status:          StatusOpen,
 			NormalizedTitle: NormalizeTitle(title),
 		}
+		if st, ok := t.triage[id]; ok {
+			issue.Status, issue.ResolvedAt = st.Status, st.ResolvedAt
+		}
 		issue.PatternSignature = patternSignature(kind, service, title, sample)
 		issue.PatternID = Fingerprint("pattern", issue.PatternSignature)
 		t.issues[id] = issue
@@ -214,6 +261,8 @@ func (t *Tracker) record(kind, service, title string, sample Sample) (string, Oc
 	if issue.Status == StatusResolved {
 		issue.Status = StatusOpen
 		issue.ResolvedAt = 0
+		issue.RegressedAt = sample.Timestamp
+		t.persistLocked(id, TriageState{Status: StatusOpen})
 	}
 	issue.Samples = append(issue.Samples, sample)
 	if len(issue.Samples) > maxSamples {
@@ -462,6 +511,8 @@ func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
 	} else {
 		issue.ResolvedAt = time.Now().UnixNano()
 	}
+	issue.RegressedAt = 0
+	t.persistLocked(id, TriageState{Status: status, ResolvedAt: issue.ResolvedAt})
 	return cloneIssue(issue), true
 }