fix
Build and Publish / Metadata (push) Successful in 9s
Build and Publish / BuildAndDeployAmd64 (push) Successful in 2m35s

This commit is contained in:
mats committed 2026-09-22 22:09:32 +02:00
1 parent e81b8c2fcd
commit 0052822e49
7 files changed
+94 -1

No files matched your search

+10
View File
@@ -244,6 +244,16 @@ func main() {
historySize := 100
store := setupStore(context.Background())
coll := collector.New(nil, historySize, store) // Temporary nil for wsServer
if occurrenceStore, ok := store.(storage.OccurrenceStore); ok {
occurrences, err := occurrenceStore.RecentOccurrences(context.Background(), 100000)
if err != nil {
log.Printf("issues: persisted index restore skipped: %v", err)
} else if n, err := coll.RestoreIssues(context.Background(), occurrences); err != nil {
log.Printf("issues: persisted index restore failed after %d occurrences: %v", n, err)
} else if len(occurrences) > 0 {
log.Printf("issues: restored %d persisted occurrences", n)
}
}
kubeWatcher, kubeErr := kube.NewFromEnv(context.Background())
if kubeErr != nil {
log.Printf("kubernetes: rollout watcher disabled: %v", kubeErr)
+17 -1
View File
@@ -312,7 +312,23 @@ func (a Alert) HTML(baseURL string, max int) string {
fmt.Fprintf(&s, "🔗 <a href=\"%s\">Open issue</a>\n", esc(base+"/issues/"+url.PathEscape(p.IssueID)))
}
if plainLen(b.String())+plainLen(s.String())+plainLen(foot)+40 > max {
fmt.Fprintf(&b, "\n… and %d more pattern%s, see the issues list", len(a.Patterns)-i, plural(len(a.Patterns)-i))
remaining := len(a.Patterns) - i
b.WriteString("\n<b>Other detected patterns</b>\n")
for _, other := range a.Patterns[i:] {
var compact strings.Builder
fmt.Fprintf(&compact, "· <code>%s</code> — %d now, normally %.0f", esc(other.What), other.Count, other.Baseline)
if base != "" && other.IssueID != "" {
fmt.Fprintf(&compact, " · <a href=\"%s\">open issue</a>", esc(base+"/issues/"+url.PathEscape(other.IssueID)))
}
compact.WriteByte('\n')
if plainLen(b.String())+plainLen(compact.String())+plainLen(foot)+10 > max {
break
}
b.WriteString(compact.String())
}
if remaining > 0 && !strings.Contains(b.String(), "open issue") && base != "" {
fmt.Fprintf(&b, "(%d additional pattern%s; open Issues for the full list)", remaining, plural(remaining))
}
break
}
b.WriteString(s.String())
+17
View File
@@ -6,6 +6,7 @@ import (
"container/ring"
"context"
"encoding/json"
"fmt"
"log"
"os"
"strconv"
@@ -230,6 +231,22 @@ func (c *Collector) SetIssueStatusStore(ctx context.Context, store issues.Status
return c.issues.SetStatusStore(ctx, store)
}
// RestoreIssues rebuilds the bounded in-memory issue index from persisted
// occurrences. This keeps /issues aligned with alerting after a restart.
func (c *Collector) RestoreIssues(ctx context.Context, occurrences []storage.Occurrence) (int, error) {
for _, occurrence := range occurrences {
var sample issues.Sample
if err := json.Unmarshal(occurrence.Sample, &sample); err != nil {
return 0, fmt.Errorf("decode persisted issue sample: %w", err)
}
if sample.Message == "" {
continue
}
c.issues.Restore(occurrence.Kind, occurrence.Service, sample.Message, sample)
}
return len(occurrences), nil
}
func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
return c.issues.Similar(id, limit)
}
+8
View File
@@ -227,6 +227,14 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
}
}
// Restore records an occurrence loaded from the persistent occurrence store.
// It deliberately does not invoke the occurrence sink, avoiding duplicate
// writes while rebuilding the in-memory index during startup.
func (t *Tracker) Restore(kind, service, title string, sample Sample) {
normalizeSample(&sample, title)
_, _ = t.record(kind, service, title, sample)
}
func (t *Tracker) record(kind, service, title string, sample Sample) (string, OccurrenceSink) {
id := Fingerprint(kind, service, NormalizeTitle(title))
bucket := HourBucket(sample.Timestamp)
+38
View File
@@ -213,6 +213,44 @@ func (s *ClickHouseStore) AppendOccurrence(ctx context.Context, occ Occurrence)
return nil
}
// RecentOccurrences returns persisted issue occurrences in chronological
// order so the in-memory issue tracker can be rebuilt after a restart.
func (s *ClickHouseStore) RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error) {
if limit <= 0 {
limit = 100000
}
rows, err := s.conn.Query(ctx, fmt.Sprintf(`
SELECT issue_id, kind, service, toUnixTimestamp64Nano(timestamp), trace_id, span_id, sample
FROM %s.issue_occurrences
ORDER BY timestamp DESC
LIMIT ?`, s.db), limit)
if err != nil {
return nil, fmt.Errorf("clickhouse recent issue occurrences: %w", err)
}
defer rows.Close()
var out []Occurrence
for rows.Next() {
var o Occurrence
var ts int64
var sample string
if err := rows.Scan(&o.IssueID, &o.Kind, &o.Service, &ts, &o.TraceID, &o.SpanID, &sample); err != nil {
return nil, fmt.Errorf("scan issue occurrence: %w", err)
}
o.Timestamp = ts
o.Sample = json.RawMessage(sample)
out = append(out, o)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate issue occurrences: %w", err)
}
// The query is newest-first; replay oldest-first to preserve issue titles
// and trend buckets as they were originally observed.
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
out[i], out[j] = out[j], out[i]
}
return out, nil
}
const maxSamplesPerIssueTrace = 20
const maxUntraced = 20
+1
View File
@@ -134,6 +134,7 @@ type IssueTracesPage struct {
// did this issue happen in" in one query (ClickHouseStore).
type OccurrenceStore interface {
AppendOccurrence(ctx context.Context, occ Occurrence) error
RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error)
IssueTraces(ctx context.Context, issueID string, opts IssueTracesOptions) (IssueTracesPage, error)
IssueSources(ctx context.Context, issueID string) ([]IssueSource, error)
}
+3
View File
@@ -264,6 +264,9 @@ rules:
- apiGroups: [""]
resources: ["pods", "events", "services"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingresses"]
verbs: ["get", "list", "watch"]