fix
This commit is contained in:
1 parent
e81b8c2fcd
commit
0052822e49
7 files changed
+94
-1
No files matched your search
@@ -244,6 +244,16 @@ func main() {
|
|||||||
historySize := 100
|
historySize := 100
|
||||||
store := setupStore(context.Background())
|
store := setupStore(context.Background())
|
||||||
coll := collector.New(nil, historySize, store) // Temporary nil for wsServer
|
coll := collector.New(nil, historySize, store) // Temporary nil for wsServer
|
||||||
|
if occurrenceStore, ok := store.(storage.OccurrenceStore); ok {
|
||||||
|
occurrences, err := occurrenceStore.RecentOccurrences(context.Background(), 100000)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("issues: persisted index restore skipped: %v", err)
|
||||||
|
} else if n, err := coll.RestoreIssues(context.Background(), occurrences); err != nil {
|
||||||
|
log.Printf("issues: persisted index restore failed after %d occurrences: %v", n, err)
|
||||||
|
} else if len(occurrences) > 0 {
|
||||||
|
log.Printf("issues: restored %d persisted occurrences", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
kubeWatcher, kubeErr := kube.NewFromEnv(context.Background())
|
kubeWatcher, kubeErr := kube.NewFromEnv(context.Background())
|
||||||
if kubeErr != nil {
|
if kubeErr != nil {
|
||||||
log.Printf("kubernetes: rollout watcher disabled: %v", kubeErr)
|
log.Printf("kubernetes: rollout watcher disabled: %v", kubeErr)
|
||||||
|
|||||||
@@ -312,7 +312,23 @@ func (a Alert) HTML(baseURL string, max int) string {
|
|||||||
fmt.Fprintf(&s, "🔗 <a href=\"%s\">Open issue</a>\n", esc(base+"/issues/"+url.PathEscape(p.IssueID)))
|
fmt.Fprintf(&s, "🔗 <a href=\"%s\">Open issue</a>\n", esc(base+"/issues/"+url.PathEscape(p.IssueID)))
|
||||||
}
|
}
|
||||||
if plainLen(b.String())+plainLen(s.String())+plainLen(foot)+40 > max {
|
if plainLen(b.String())+plainLen(s.String())+plainLen(foot)+40 > max {
|
||||||
fmt.Fprintf(&b, "\n… and %d more pattern%s, see the issues list", len(a.Patterns)-i, plural(len(a.Patterns)-i))
|
remaining := len(a.Patterns) - i
|
||||||
|
b.WriteString("\n<b>Other detected patterns</b>\n")
|
||||||
|
for _, other := range a.Patterns[i:] {
|
||||||
|
var compact strings.Builder
|
||||||
|
fmt.Fprintf(&compact, "· <code>%s</code> — %d now, normally %.0f", esc(other.What), other.Count, other.Baseline)
|
||||||
|
if base != "" && other.IssueID != "" {
|
||||||
|
fmt.Fprintf(&compact, " · <a href=\"%s\">open issue</a>", esc(base+"/issues/"+url.PathEscape(other.IssueID)))
|
||||||
|
}
|
||||||
|
compact.WriteByte('\n')
|
||||||
|
if plainLen(b.String())+plainLen(compact.String())+plainLen(foot)+10 > max {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
b.WriteString(compact.String())
|
||||||
|
}
|
||||||
|
if remaining > 0 && !strings.Contains(b.String(), "open issue") && base != "" {
|
||||||
|
fmt.Fprintf(&b, "(%d additional pattern%s; open Issues for the full list)", remaining, plural(remaining))
|
||||||
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
b.WriteString(s.String())
|
b.WriteString(s.String())
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"container/ring"
|
"container/ring"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -230,6 +231,22 @@ func (c *Collector) SetIssueStatusStore(ctx context.Context, store issues.Status
|
|||||||
return c.issues.SetStatusStore(ctx, store)
|
return c.issues.SetStatusStore(ctx, store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RestoreIssues rebuilds the bounded in-memory issue index from persisted
|
||||||
|
// occurrences. This keeps /issues aligned with alerting after a restart.
|
||||||
|
func (c *Collector) RestoreIssues(ctx context.Context, occurrences []storage.Occurrence) (int, error) {
|
||||||
|
for _, occurrence := range occurrences {
|
||||||
|
var sample issues.Sample
|
||||||
|
if err := json.Unmarshal(occurrence.Sample, &sample); err != nil {
|
||||||
|
return 0, fmt.Errorf("decode persisted issue sample: %w", err)
|
||||||
|
}
|
||||||
|
if sample.Message == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.issues.Restore(occurrence.Kind, occurrence.Service, sample.Message, sample)
|
||||||
|
}
|
||||||
|
return len(occurrences), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
|
func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
|
||||||
return c.issues.Similar(id, limit)
|
return c.issues.Similar(id, limit)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -227,6 +227,14 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Restore records an occurrence loaded from the persistent occurrence store.
|
||||||
|
// It deliberately does not invoke the occurrence sink, avoiding duplicate
|
||||||
|
// writes while rebuilding the in-memory index during startup.
|
||||||
|
func (t *Tracker) Restore(kind, service, title string, sample Sample) {
|
||||||
|
normalizeSample(&sample, title)
|
||||||
|
_, _ = t.record(kind, service, title, sample)
|
||||||
|
}
|
||||||
|
|
||||||
func (t *Tracker) record(kind, service, title string, sample Sample) (string, OccurrenceSink) {
|
func (t *Tracker) record(kind, service, title string, sample Sample) (string, OccurrenceSink) {
|
||||||
id := Fingerprint(kind, service, NormalizeTitle(title))
|
id := Fingerprint(kind, service, NormalizeTitle(title))
|
||||||
bucket := HourBucket(sample.Timestamp)
|
bucket := HourBucket(sample.Timestamp)
|
||||||
|
|||||||
@@ -213,6 +213,44 @@ func (s *ClickHouseStore) AppendOccurrence(ctx context.Context, occ Occurrence)
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecentOccurrences returns persisted issue occurrences in chronological
|
||||||
|
// order so the in-memory issue tracker can be rebuilt after a restart.
|
||||||
|
func (s *ClickHouseStore) RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 100000
|
||||||
|
}
|
||||||
|
rows, err := s.conn.Query(ctx, fmt.Sprintf(`
|
||||||
|
SELECT issue_id, kind, service, toUnixTimestamp64Nano(timestamp), trace_id, span_id, sample
|
||||||
|
FROM %s.issue_occurrences
|
||||||
|
ORDER BY timestamp DESC
|
||||||
|
LIMIT ?`, s.db), limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clickhouse recent issue occurrences: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Occurrence
|
||||||
|
for rows.Next() {
|
||||||
|
var o Occurrence
|
||||||
|
var ts int64
|
||||||
|
var sample string
|
||||||
|
if err := rows.Scan(&o.IssueID, &o.Kind, &o.Service, &ts, &o.TraceID, &o.SpanID, &sample); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan issue occurrence: %w", err)
|
||||||
|
}
|
||||||
|
o.Timestamp = ts
|
||||||
|
o.Sample = json.RawMessage(sample)
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("iterate issue occurrences: %w", err)
|
||||||
|
}
|
||||||
|
// The query is newest-first; replay oldest-first to preserve issue titles
|
||||||
|
// and trend buckets as they were originally observed.
|
||||||
|
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
out[i], out[j] = out[j], out[i]
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
const maxSamplesPerIssueTrace = 20
|
const maxSamplesPerIssueTrace = 20
|
||||||
const maxUntraced = 20
|
const maxUntraced = 20
|
||||||
|
|
||||||
|
|||||||
@@ -134,6 +134,7 @@ type IssueTracesPage struct {
|
|||||||
// did this issue happen in" in one query (ClickHouseStore).
|
// did this issue happen in" in one query (ClickHouseStore).
|
||||||
type OccurrenceStore interface {
|
type OccurrenceStore interface {
|
||||||
AppendOccurrence(ctx context.Context, occ Occurrence) error
|
AppendOccurrence(ctx context.Context, occ Occurrence) error
|
||||||
|
RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error)
|
||||||
IssueTraces(ctx context.Context, issueID string, opts IssueTracesOptions) (IssueTracesPage, error)
|
IssueTraces(ctx context.Context, issueID string, opts IssueTracesOptions) (IssueTracesPage, error)
|
||||||
IssueSources(ctx context.Context, issueID string) ([]IssueSource, error)
|
IssueSources(ctx context.Context, issueID string) ([]IssueSource, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -264,6 +264,9 @@ rules:
|
|||||||
- apiGroups: [""]
|
- apiGroups: [""]
|
||||||
resources: ["pods", "events", "services"]
|
resources: ["pods", "events", "services"]
|
||||||
verbs: ["get", "list", "watch"]
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods/log"]
|
||||||
|
verbs: ["get"]
|
||||||
- apiGroups: ["networking.k8s.io"]
|
- apiGroups: ["networking.k8s.io"]
|
||||||
resources: ["ingresses"]
|
resources: ["ingresses"]
|
||||||
verbs: ["get", "list", "watch"]
|
verbs: ["get", "list", "watch"]
|
||||||
|
|||||||
Reference in new issue
Block a user