include db for gitea build
This commit is contained in:
mats committed
1 parent
8de4a2e010
commit
645f923175
33 files changed
+1490
-150
No files matched your search
+26
-91
@@ -17,8 +17,10 @@ import (
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/alerts"
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
"git.tornberg.me/go-otel/internal/triage"
|
||||
"git.tornberg.me/go-otel/internal/uaclass"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -51,18 +53,28 @@ func main() {
|
||||
}
|
||||
cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}
|
||||
|
||||
events := load(*addr, *database, *user, *password, start, end)
|
||||
cfg.DataSince = dataSince(*addr, *database, *user, *password)
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
events, err := alerts.LoadErrorEvents(ctx, conn, *database, start, end)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
cfg.DataSince = alerts.DataSince(ctx, conn, *database)
|
||||
log.Printf("loaded %d error events", len(events))
|
||||
|
||||
cache := map[string]classification{}
|
||||
first := start.Add(time.Duration(*baselineWindows) * *window)
|
||||
suppress := features.NewSuppressor(*cooldown)
|
||||
muted := map[triage.Category]int{}
|
||||
alerts := 0
|
||||
fired := 0
|
||||
for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
|
||||
for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
|
||||
alerts++
|
||||
for _, spike := range suppress.Filter(alerts.FilterActionable(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
|
||||
fired++
|
||||
var cls []classification
|
||||
for _, p := range spike.Patterns {
|
||||
c := classification{}
|
||||
@@ -86,7 +98,7 @@ func main() {
|
||||
for c, n := range muted {
|
||||
log.Printf("muted %d pattern-window(s) as %s", n, c)
|
||||
}
|
||||
log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
||||
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
type classification struct {
|
||||
@@ -96,35 +108,6 @@ type classification struct {
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func triageOf(e features.Event) triage.Result {
|
||||
return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
|
||||
IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
|
||||
}
|
||||
|
||||
// triageFilter keeps only actionable patterns (unless showMuted), recomputing
|
||||
// each spike's excess so a probe storm cannot page anyone by itself.
|
||||
func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
|
||||
var out []features.Spike
|
||||
for _, sp := range spikes {
|
||||
var kept []features.PatternDelta
|
||||
sp.Excess = 0
|
||||
for _, p := range sp.Patterns {
|
||||
c := triageOf(p.Example).Category
|
||||
if !showMuted && !c.Actionable() {
|
||||
muted[c]++
|
||||
continue
|
||||
}
|
||||
kept = append(kept, p)
|
||||
sp.Excess += p.Excess
|
||||
}
|
||||
if len(kept) > 0 {
|
||||
sp.Patterns = kept
|
||||
out = append(out, sp)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func classify(base, text string) classification {
|
||||
body, _ := json.Marshal(map[string]string{"text": text})
|
||||
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
|
||||
@@ -163,47 +146,19 @@ func printSpike(s features.Spike, cls []classification) {
|
||||
if p.Novel {
|
||||
tag = "NEW"
|
||||
}
|
||||
what := string(triageOf(p.Example).Category)
|
||||
what := string(alerts.TriageOf(p.Example).Category)
|
||||
if cls[i].Action != "" {
|
||||
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
|
||||
}
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
|
||||
}
|
||||
}
|
||||
|
||||
func load(addr, database, user, password string, start, end time.Time) []features.Event {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
// Errors only, and only columns present in every deployed schema version.
|
||||
rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
|
||||
FROM %s.normalized_events FINAL
|
||||
WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
|
||||
ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []features.Event
|
||||
for rows.Next() {
|
||||
e := features.Event{IsError: true}
|
||||
var status uint16
|
||||
var timeout, cancel, retry uint8
|
||||
if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
|
||||
log.Fatal(err)
|
||||
clients := ""
|
||||
if p.Clients > 0 {
|
||||
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
|
||||
}
|
||||
e.StatusCode = int(status)
|
||||
e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
|
||||
out = append(out, e)
|
||||
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
||||
clients += " [" + p.Example.UAClass + "]"
|
||||
}
|
||||
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
|
||||
@@ -213,23 +168,3 @@ func envOr(k, f string) string {
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// dataSince returns the earliest normalized row, used as the warm-up guard.
|
||||
func dataSince(addr, database, user, password string) time.Time {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
var t time.Time
|
||||
if err := conn.QueryRow(ctx, fmt.Sprintf(`SELECT min(timestamp) FROM %s.normalized_events`, database)).Scan(&t); err != nil {
|
||||
log.Printf("warm-up guard disabled: %v", err)
|
||||
return time.Time{}
|
||||
}
|
||||
if t.Year() < 2000 { // empty table
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
Reference in new issue
Block a user