developer
8d71ef06c6
Adds internal/laya, a client and safety-gated decision engine for the Laya decision API (https://laya.k6n.net). The API's OpenAPI document leaves every response schema empty, so the wire format was pinned by probing the live service: questions are typed (choice/score/noul) and carry "criteria" as an object for a choice and an array for a score. An alert is rendered into a compact factual state and asked four typed questions: which action (restart/ignore/notify/escalate), severity and how urgent. The state always includes the pods' last events - restart count, readiness, last termination reason and exit code, log tail, and any overlapping rollout - because a 503 burst from a pod being replaced is otherwise indistinguishable from an outage. The model proposes, the policy disposes. Every gate records why it changed an answer: an executable-action allowlist, a probability and lead-over-runner-up threshold, no restart while pods are crash-looping (the kubelet is already doing it) or while a rollout is replacing every pod (the 503s are the rollout's own), never ignoring a new error or a high-priority spike without a human's mute, an hourly restart budget per deployment, and a plain notification whenever the model is unreachable. The margin, not the raw probability, is the useful signal: label probabilities are diffuse and a correct answer rarely exceeds 0.5. topology.PodLog now carries the pod's lifecycle facts alongside the log tail. PodsNotReady/CrashLooping helpers build on them. Verified end to end against the live API: a crash loop is withheld as "already crash-looping", a rollout's 503 burst is held back at a 2% margin, and crawler 404s are ignored at 13%. LAYA_URL=... go test ./internal/laya -run TestLiveModel replays it.
60 lines
2.9 KiB
Go
60 lines
2.9 KiB
Go
package laya
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// TestLiveModel is a smoke test against the real Laya API, so the state
|
|
// rendering and the questions stay in step with what the model actually
|
|
// understands. It is opt-in: LAYA_URL=https://laya.k6n.net go test ./internal/laya -run TestLiveModel
|
|
func TestLiveModel(t *testing.T) {
|
|
base := os.Getenv("LAYA_URL")
|
|
if base == "" {
|
|
t.Skip("set LAYA_URL to run against the live model")
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
in Input
|
|
}{
|
|
{
|
|
name: "crash loop after a release",
|
|
in: Input{
|
|
Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute, BaselineMean: 3, NewErrors: true, Clients: 42,
|
|
Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic: runtime error: invalid memory address", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42, Services: []string{"checkout-api", "cart-service"}}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running", Restarts: 3, PodRestarts: 9, Reason: "CrashLoopBackOff", LastTerminatedAt: time.Now().Add(-30 * time.Second), LastEvents: []string{"panic: runtime error: invalid memory address", "SIGTERM received"}}},
|
|
Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Release: "v2.31", Age: 4 * time.Minute, Ready: 1, Desired: 3}},
|
|
},
|
|
},
|
|
{
|
|
name: "503 burst while a pod is being replaced",
|
|
in: Input{
|
|
Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2, Clients: 8,
|
|
Patterns: []Pattern{{What: "HTTP 503 | /api/cart | no healthy upstream", Category: "capacity-saturation", Status: 503, Count: 40, Baseline: 1, Clients: 8}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-2k1", Found: true, Phase: "Pending", LastTerminatedAt: time.Now().Add(-40 * time.Second), LastEvents: []string{"terminated"}}},
|
|
Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}},
|
|
},
|
|
},
|
|
{
|
|
name: "crawler 404s, nobody affected",
|
|
in: Input{
|
|
Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true,
|
|
Patterns: []Pattern{{What: "HTTP 404 | /old-post-2019 | UA looks like a crawler", Status: 404, Count: 1800, Baseline: 15}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "blog", Name: "blog-6f2", Found: true, Ready: true, Phase: "Running", LastEvents: []string{"GET /old-post-2019 404"}}},
|
|
},
|
|
},
|
|
}
|
|
// Engine defaults, as deployed: the label probabilities are diffuse, so the
|
|
// gate is the lead over the runner-up.
|
|
engine := NewEngine(NewClient(base), Policy{Actions: Actions, Now: time.Now})
|
|
for _, tc := range cases {
|
|
v := engine.Decide(context.Background(), tc.in)
|
|
t.Logf("%s:\n%s\n -> %s\n", tc.name, v.State, v.Explain())
|
|
if v.Error != "" {
|
|
t.Errorf("%s: %v", tc.name, v.Error)
|
|
}
|
|
}
|
|
}
|