fix
This commit is contained in:
1 parent
e81b8c2fcd
commit
0052822e49
7 files changed
+94
-1
No files matched your search
@@ -312,7 +312,23 @@ func (a Alert) HTML(baseURL string, max int) string {
|
||||
fmt.Fprintf(&s, "🔗 <a href=\"%s\">Open issue</a>\n", esc(base+"/issues/"+url.PathEscape(p.IssueID)))
|
||||
}
|
||||
if plainLen(b.String())+plainLen(s.String())+plainLen(foot)+40 > max {
|
||||
fmt.Fprintf(&b, "\n… and %d more pattern%s, see the issues list", len(a.Patterns)-i, plural(len(a.Patterns)-i))
|
||||
remaining := len(a.Patterns) - i
|
||||
b.WriteString("\n<b>Other detected patterns</b>\n")
|
||||
for _, other := range a.Patterns[i:] {
|
||||
var compact strings.Builder
|
||||
fmt.Fprintf(&compact, "· <code>%s</code> — %d now, normally %.0f", esc(other.What), other.Count, other.Baseline)
|
||||
if base != "" && other.IssueID != "" {
|
||||
fmt.Fprintf(&compact, " · <a href=\"%s\">open issue</a>", esc(base+"/issues/"+url.PathEscape(other.IssueID)))
|
||||
}
|
||||
compact.WriteByte('\n')
|
||||
if plainLen(b.String())+plainLen(compact.String())+plainLen(foot)+10 > max {
|
||||
break
|
||||
}
|
||||
b.WriteString(compact.String())
|
||||
}
|
||||
if remaining > 0 && !strings.Contains(b.String(), "open issue") && base != "" {
|
||||
fmt.Fprintf(&b, "(%d additional pattern%s; open Issues for the full list)", remaining, plural(remaining))
|
||||
}
|
||||
break
|
||||
}
|
||||
b.WriteString(s.String())
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"container/ring"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
@@ -230,6 +231,22 @@ func (c *Collector) SetIssueStatusStore(ctx context.Context, store issues.Status
|
||||
return c.issues.SetStatusStore(ctx, store)
|
||||
}
|
||||
|
||||
// RestoreIssues rebuilds the bounded in-memory issue index from persisted
|
||||
// occurrences. This keeps /issues aligned with alerting after a restart.
|
||||
func (c *Collector) RestoreIssues(ctx context.Context, occurrences []storage.Occurrence) (int, error) {
|
||||
for _, occurrence := range occurrences {
|
||||
var sample issues.Sample
|
||||
if err := json.Unmarshal(occurrence.Sample, &sample); err != nil {
|
||||
return 0, fmt.Errorf("decode persisted issue sample: %w", err)
|
||||
}
|
||||
if sample.Message == "" {
|
||||
continue
|
||||
}
|
||||
c.issues.Restore(occurrence.Kind, occurrence.Service, sample.Message, sample)
|
||||
}
|
||||
return len(occurrences), nil
|
||||
}
|
||||
|
||||
func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
|
||||
return c.issues.Similar(id, limit)
|
||||
}
|
||||
|
||||
@@ -227,6 +227,14 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
|
||||
}
|
||||
}
|
||||
|
||||
// Restore records an occurrence loaded from the persistent occurrence store.
|
||||
// It deliberately does not invoke the occurrence sink, avoiding duplicate
|
||||
// writes while rebuilding the in-memory index during startup.
|
||||
func (t *Tracker) Restore(kind, service, title string, sample Sample) {
|
||||
normalizeSample(&sample, title)
|
||||
_, _ = t.record(kind, service, title, sample)
|
||||
}
|
||||
|
||||
func (t *Tracker) record(kind, service, title string, sample Sample) (string, OccurrenceSink) {
|
||||
id := Fingerprint(kind, service, NormalizeTitle(title))
|
||||
bucket := HourBucket(sample.Timestamp)
|
||||
|
||||
@@ -213,6 +213,44 @@ func (s *ClickHouseStore) AppendOccurrence(ctx context.Context, occ Occurrence)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecentOccurrences returns persisted issue occurrences in chronological
|
||||
// order so the in-memory issue tracker can be rebuilt after a restart.
|
||||
func (s *ClickHouseStore) RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100000
|
||||
}
|
||||
rows, err := s.conn.Query(ctx, fmt.Sprintf(`
|
||||
SELECT issue_id, kind, service, toUnixTimestamp64Nano(timestamp), trace_id, span_id, sample
|
||||
FROM %s.issue_occurrences
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?`, s.db), limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clickhouse recent issue occurrences: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Occurrence
|
||||
for rows.Next() {
|
||||
var o Occurrence
|
||||
var ts int64
|
||||
var sample string
|
||||
if err := rows.Scan(&o.IssueID, &o.Kind, &o.Service, &ts, &o.TraceID, &o.SpanID, &sample); err != nil {
|
||||
return nil, fmt.Errorf("scan issue occurrence: %w", err)
|
||||
}
|
||||
o.Timestamp = ts
|
||||
o.Sample = json.RawMessage(sample)
|
||||
out = append(out, o)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate issue occurrences: %w", err)
|
||||
}
|
||||
// The query is newest-first; replay oldest-first to preserve issue titles
|
||||
// and trend buckets as they were originally observed.
|
||||
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
||||
out[i], out[j] = out[j], out[i]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
const maxSamplesPerIssueTrace = 20
|
||||
const maxUntraced = 20
|
||||
|
||||
|
||||
@@ -134,6 +134,7 @@ type IssueTracesPage struct {
|
||||
// did this issue happen in" in one query (ClickHouseStore).
|
||||
type OccurrenceStore interface {
|
||||
AppendOccurrence(ctx context.Context, occ Occurrence) error
|
||||
RecentOccurrences(ctx context.Context, limit int) ([]Occurrence, error)
|
||||
IssueTraces(ctx context.Context, issueID string, opts IssueTracesOptions) (IssueTracesPage, error)
|
||||
IssueSources(ctx context.Context, issueID string) ([]IssueSource, error)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user