316 lines
10 KiB
Go
316 lines
10 KiB
Go
// Package kubernetes observes read-only workload state for rollout correlation.
|
|
package kubernetes
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/informers"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/tools/cache"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
)
|
|
|
|
const maxRollouts = 500
|
|
|
|
type Watcher struct {
|
|
mu sync.RWMutex
|
|
rollouts map[string]Rollout
|
|
// revisions maps "namespace/deployment/revision" to the creation time of
|
|
// the ReplicaSet backing that revision - the cluster's own record of when
|
|
// each rollout started, so times survive watcher restarts.
|
|
revisions map[string]int64
|
|
enabled bool
|
|
err string
|
|
}
|
|
|
|
// NewFromEnv enables observation in-cluster, or with KUBECONFIG locally. It is
|
|
// disabled by default outside Kubernetes so local development needs no kubeconfig.
|
|
func NewFromEnv(ctx context.Context) (*Watcher, error) {
|
|
if strings.EqualFold(os.Getenv("OTEL_KUBERNETES_ENABLED"), "false") {
|
|
return nil, nil
|
|
}
|
|
config, err := kubeConfig()
|
|
if err != nil {
|
|
if os.Getenv("OTEL_KUBERNETES_ENABLED") == "true" {
|
|
return nil, err
|
|
}
|
|
return nil, nil
|
|
}
|
|
client, err := kubernetes.NewForConfig(config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create kubernetes client: %w", err)
|
|
}
|
|
|
|
w := &Watcher{rollouts: make(map[string]Rollout), revisions: make(map[string]int64), enabled: true}
|
|
namespace := os.Getenv("OTEL_KUBERNETES_NAMESPACE")
|
|
selector := os.Getenv("OTEL_KUBERNETES_LABEL_SELECTOR")
|
|
factory := informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
|
|
informers.WithTweakListOptions(func(options *metav1.ListOptions) {
|
|
options.LabelSelector = selector
|
|
}),
|
|
)
|
|
if namespace != "" {
|
|
factory = informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
|
|
informers.WithNamespace(namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) {
|
|
options.LabelSelector = selector
|
|
}))
|
|
}
|
|
informer := factory.Apps().V1().Deployments().Informer()
|
|
_, _ = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) { w.observe(obj) },
|
|
UpdateFunc: func(_, obj interface{}) { w.observe(obj) },
|
|
})
|
|
rsInformer := factory.Apps().V1().ReplicaSets().Informer()
|
|
_, _ = rsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) { w.observeReplicaSet(obj) },
|
|
UpdateFunc: func(_, obj interface{}) { w.observeReplicaSet(obj) },
|
|
})
|
|
go func() {
|
|
factory.Start(ctx.Done())
|
|
if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced, rsInformer.HasSynced) {
|
|
w.setError("deployment informer did not sync")
|
|
return
|
|
}
|
|
<-ctx.Done()
|
|
}()
|
|
return w, nil
|
|
}
|
|
|
|
func kubeConfig() (*rest.Config, error) {
|
|
if config, err := rest.InClusterConfig(); err == nil {
|
|
return config, nil
|
|
}
|
|
kubeconfig := os.Getenv("KUBECONFIG")
|
|
if kubeconfig == "" {
|
|
home, err := os.UserHomeDir()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kubeconfig = home + string(os.PathSeparator) + ".kube" + string(os.PathSeparator) + "config"
|
|
}
|
|
return clientcmd.BuildConfigFromFlags("", kubeconfig)
|
|
}
|
|
|
|
func (w *Watcher) observe(obj interface{}) {
|
|
deployment, ok := obj.(*appsv1.Deployment)
|
|
if !ok {
|
|
return
|
|
}
|
|
now := time.Now().UnixNano()
|
|
revision := deployment.Annotations["deployment.kubernetes.io/revision"]
|
|
if revision == "" {
|
|
revision = strconv.FormatInt(deployment.Generation, 10)
|
|
}
|
|
key := deployment.Namespace + "/" + deployment.Name + "/" + revision
|
|
phase, reason, message, completed := rolloutStatus(deployment)
|
|
|
|
// StartedAt is when the rollout began. The ReplicaSet for this revision was
|
|
// created at that moment, so prefer the cluster's own timestamp; it is also
|
|
// correct for rollouts first observed long after they happened. Fall back
|
|
// to first-seen via the Progressing condition, then to this event.
|
|
started := w.revisionCreated(deployment.Namespace, deployment.Name, revision)
|
|
if started == 0 {
|
|
if old, exists := w.rollouts[key]; exists && old.StartedAt != 0 {
|
|
started = old.StartedAt
|
|
} else {
|
|
for _, condition := range deployment.Status.Conditions {
|
|
if condition.Type == appsv1.DeploymentProgressing && !condition.LastUpdateTime.IsZero() {
|
|
started = condition.LastUpdateTime.UnixNano()
|
|
}
|
|
}
|
|
if started == 0 {
|
|
started = now
|
|
}
|
|
}
|
|
}
|
|
// CompletedAt prefers the Progressing condition's own update time for a
|
|
// finished rollout, and once set is never re-bumped: a completed rollout
|
|
// stays completed at the moment it completed.
|
|
completedAt := int64(0)
|
|
if completed {
|
|
for _, condition := range deployment.Status.Conditions {
|
|
if condition.Type == appsv1.DeploymentProgressing && condition.Status == corev1.ConditionTrue && condition.Reason == "NewReplicaSetAvailable" {
|
|
completedAt = condition.LastUpdateTime.UnixNano()
|
|
}
|
|
}
|
|
if completedAt == 0 {
|
|
if old, exists := w.rollouts[key]; exists && old.CompletedAt != 0 {
|
|
completedAt = old.CompletedAt
|
|
} else {
|
|
completedAt = now
|
|
}
|
|
}
|
|
}
|
|
rollout := Rollout{
|
|
ID: rolloutID(key), Namespace: deployment.Namespace, Workload: deployment.Name,
|
|
Service: metadataValue(deployment, "app.kubernetes.io/name", "app", "service.name"),
|
|
Release: metadataValue(deployment, "otel.k6n.net/release", "app.kubernetes.io/version", "service.release"),
|
|
Revision: revision, Image: deploymentImage(deployment), Phase: phase, Reason: reason,
|
|
Message: message, StartedAt: started, LastSeen: now, Generation: deployment.Generation,
|
|
DesiredReplicas: desiredReplicas(deployment), UpdatedReplicas: deployment.Status.UpdatedReplicas,
|
|
AvailableReplicas: deployment.Status.AvailableReplicas, ReadyReplicas: deployment.Status.ReadyReplicas,
|
|
CompletedAt: completedAt,
|
|
}
|
|
w.mu.Lock()
|
|
// The informer re-delivers every deployment on each resync (~30s). Only an
|
|
// observable change advances LastSeen, so a quiet rollout keeps its real
|
|
// last-changed time instead of looking perpetually fresh.
|
|
if old, exists := w.rollouts[key]; exists && !rolloutChanged(old, rollout) {
|
|
rollout.LastSeen = old.LastSeen
|
|
}
|
|
w.rollouts[key] = rollout
|
|
if len(w.rollouts) > maxRollouts {
|
|
w.evictOldestLocked()
|
|
}
|
|
w.mu.Unlock()
|
|
}
|
|
|
|
func (w *Watcher) observeReplicaSet(obj interface{}) {
|
|
rs, ok := obj.(*appsv1.ReplicaSet)
|
|
if !ok {
|
|
return
|
|
}
|
|
revision := rs.Annotations["deployment.kubernetes.io/revision"]
|
|
owner := ""
|
|
for _, ref := range rs.OwnerReferences {
|
|
if ref.Kind == "Deployment" && ref.Name != "" {
|
|
owner = ref.Name
|
|
}
|
|
}
|
|
if revision == "" || owner == "" || rs.CreationTimestamp.IsZero() {
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
w.revisions[rs.Namespace+"/"+owner+"/"+revision] = rs.CreationTimestamp.UnixNano()
|
|
w.mu.Unlock()
|
|
}
|
|
|
|
func (w *Watcher) revisionCreated(namespace, workload, revision string) int64 {
|
|
w.mu.RLock()
|
|
defer w.mu.RUnlock()
|
|
return w.revisions[namespace+"/"+workload+"/"+revision]
|
|
}
|
|
|
|
// rolloutChanged reports whether anything observable about the rollout moved.
|
|
func rolloutChanged(a, b Rollout) bool {
|
|
return a.Phase != b.Phase || a.Reason != b.Reason || a.Message != b.Message ||
|
|
a.Service != b.Service || a.Release != b.Release || a.Image != b.Image ||
|
|
a.Generation != b.Generation ||
|
|
a.DesiredReplicas != b.DesiredReplicas || a.UpdatedReplicas != b.UpdatedReplicas ||
|
|
a.AvailableReplicas != b.AvailableReplicas || a.ReadyReplicas != b.ReadyReplicas
|
|
}
|
|
|
|
func (w *Watcher) evictOldestLocked() {
|
|
var oldest string
|
|
var ts int64
|
|
for key, item := range w.rollouts {
|
|
if oldest == "" || item.StartedAt < ts {
|
|
oldest, ts = key, item.StartedAt
|
|
}
|
|
}
|
|
if oldest != "" {
|
|
delete(w.rollouts, oldest)
|
|
}
|
|
}
|
|
func (w *Watcher) setError(message string) { w.mu.Lock(); defer w.mu.Unlock(); w.err = message }
|
|
|
|
func (w *Watcher) List() []Rollout {
|
|
w.mu.RLock()
|
|
defer w.mu.RUnlock()
|
|
result := make([]Rollout, 0, len(w.rollouts))
|
|
for _, item := range w.rollouts {
|
|
result = append(result, item)
|
|
}
|
|
sort.Slice(result, func(i, j int) bool {
|
|
if result[i].StartedAt != result[j].StartedAt {
|
|
return result[i].StartedAt > result[j].StartedAt
|
|
}
|
|
return result[i].LastSeen > result[j].LastSeen
|
|
})
|
|
return result
|
|
}
|
|
func (w *Watcher) Get(id string) (Rollout, bool) {
|
|
w.mu.RLock()
|
|
defer w.mu.RUnlock()
|
|
for _, item := range w.rollouts {
|
|
if item.ID == id {
|
|
return item, true
|
|
}
|
|
}
|
|
return Rollout{}, false
|
|
}
|
|
func (w *Watcher) Enabled() bool { return w != nil && w.enabled }
|
|
func (w *Watcher) Error() string {
|
|
if w == nil {
|
|
return "kubernetes integration is not configured"
|
|
}
|
|
w.mu.RLock()
|
|
defer w.mu.RUnlock()
|
|
return w.err
|
|
}
|
|
|
|
func rolloutID(key string) string {
|
|
sum := sha256.Sum256([]byte(key))
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
func metadataValue(deployment *appsv1.Deployment, keys ...string) string {
|
|
for _, key := range keys {
|
|
if value := deployment.Labels[key]; value != "" {
|
|
return value
|
|
}
|
|
if value := deployment.Annotations[key]; value != "" {
|
|
return value
|
|
}
|
|
if value := deployment.Spec.Template.Labels[key]; value != "" {
|
|
return value
|
|
}
|
|
if value := deployment.Spec.Template.Annotations[key]; value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
func deploymentImage(deployment *appsv1.Deployment) string {
|
|
if len(deployment.Spec.Template.Spec.Containers) == 0 {
|
|
return ""
|
|
}
|
|
return deployment.Spec.Template.Spec.Containers[0].Image
|
|
}
|
|
func desiredReplicas(deployment *appsv1.Deployment) int32 {
|
|
if deployment.Spec.Replicas == nil {
|
|
return 1
|
|
}
|
|
return *deployment.Spec.Replicas
|
|
}
|
|
|
|
func rolloutStatus(deployment *appsv1.Deployment) (phase, reason, message string, completed bool) {
|
|
phase = "progressing"
|
|
for _, condition := range deployment.Status.Conditions {
|
|
switch condition.Type {
|
|
case appsv1.DeploymentProgressing:
|
|
reason, message = condition.Reason, condition.Message
|
|
if condition.Status == corev1.ConditionFalse {
|
|
return "failed", reason, message, false
|
|
}
|
|
case appsv1.DeploymentAvailable:
|
|
if condition.Status == corev1.ConditionTrue && deployment.Status.UpdatedReplicas == desiredReplicas(deployment) && deployment.Status.AvailableReplicas >= desiredReplicas(deployment) {
|
|
return "completed", reason, message, true
|
|
}
|
|
}
|
|
}
|
|
return phase, reason, message, false
|
|
}
|