// Package kubernetes observes read-only workload state for rollout correlation. package kubernetes import ( "context" "crypto/sha256" "encoding/hex" "fmt" "os" "sort" "strconv" "strings" "sync" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" ) const maxRollouts = 500 type Watcher struct { mu sync.RWMutex rollouts map[string]Rollout // revisions maps "namespace/deployment/revision" to the creation time of // the ReplicaSet backing that revision - the cluster's own record of when // each rollout started, so times survive watcher restarts. revisions map[string]int64 enabled bool err string } // NewFromEnv enables observation in-cluster, or with KUBECONFIG locally. It is // disabled by default outside Kubernetes so local development needs no kubeconfig. func NewFromEnv(ctx context.Context) (*Watcher, error) { if strings.EqualFold(os.Getenv("OTEL_KUBERNETES_ENABLED"), "false") { return nil, nil } config, err := kubeConfig() if err != nil { if os.Getenv("OTEL_KUBERNETES_ENABLED") == "true" { return nil, err } return nil, nil } client, err := kubernetes.NewForConfig(config) if err != nil { return nil, fmt.Errorf("create kubernetes client: %w", err) } w := &Watcher{rollouts: make(map[string]Rollout), revisions: make(map[string]int64), enabled: true} namespace := os.Getenv("OTEL_KUBERNETES_NAMESPACE") selector := os.Getenv("OTEL_KUBERNETES_LABEL_SELECTOR") factory := informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = selector }), ) if namespace != "" { factory = informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second, informers.WithNamespace(namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = selector })) } informer := factory.Apps().V1().Deployments().Informer() _, _ = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { w.observe(obj) }, UpdateFunc: func(_, obj interface{}) { w.observe(obj) }, }) rsInformer := factory.Apps().V1().ReplicaSets().Informer() _, _ = rsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { w.observeReplicaSet(obj) }, UpdateFunc: func(_, obj interface{}) { w.observeReplicaSet(obj) }, }) go func() { factory.Start(ctx.Done()) if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced, rsInformer.HasSynced) { w.setError("deployment informer did not sync") return } <-ctx.Done() }() return w, nil } func kubeConfig() (*rest.Config, error) { if config, err := rest.InClusterConfig(); err == nil { return config, nil } kubeconfig := os.Getenv("KUBECONFIG") if kubeconfig == "" { home, err := os.UserHomeDir() if err != nil { return nil, err } kubeconfig = home + string(os.PathSeparator) + ".kube" + string(os.PathSeparator) + "config" } return clientcmd.BuildConfigFromFlags("", kubeconfig) } func (w *Watcher) observe(obj interface{}) { deployment, ok := obj.(*appsv1.Deployment) if !ok { return } now := time.Now().UnixNano() revision := deployment.Annotations["deployment.kubernetes.io/revision"] if revision == "" { revision = strconv.FormatInt(deployment.Generation, 10) } key := deployment.Namespace + "/" + deployment.Name + "/" + revision phase, reason, message, completed := rolloutStatus(deployment) // StartedAt is when the rollout began. The ReplicaSet for this revision was // created at that moment, so prefer the cluster's own timestamp; it is also // correct for rollouts first observed long after they happened. Fall back // to first-seen via the Progressing condition, then to this event. started := w.revisionCreated(deployment.Namespace, deployment.Name, revision) if started == 0 { if old, exists := w.rollouts[key]; exists && old.StartedAt != 0 { started = old.StartedAt } else { for _, condition := range deployment.Status.Conditions { if condition.Type == appsv1.DeploymentProgressing && !condition.LastUpdateTime.IsZero() { started = condition.LastUpdateTime.UnixNano() } } if started == 0 { started = now } } } // CompletedAt prefers the Progressing condition's own update time for a // finished rollout, and once set is never re-bumped: a completed rollout // stays completed at the moment it completed. completedAt := int64(0) if completed { for _, condition := range deployment.Status.Conditions { if condition.Type == appsv1.DeploymentProgressing && condition.Status == corev1.ConditionTrue && condition.Reason == "NewReplicaSetAvailable" { completedAt = condition.LastUpdateTime.UnixNano() } } if completedAt == 0 { if old, exists := w.rollouts[key]; exists && old.CompletedAt != 0 { completedAt = old.CompletedAt } else { completedAt = now } } } rollout := Rollout{ ID: rolloutID(key), Namespace: deployment.Namespace, Workload: deployment.Name, Service: metadataValue(deployment, "app.kubernetes.io/name", "app", "service.name"), Release: metadataValue(deployment, "otel.k6n.net/release", "app.kubernetes.io/version", "service.release"), Revision: revision, Image: deploymentImage(deployment), Phase: phase, Reason: reason, Message: message, StartedAt: started, LastSeen: now, Generation: deployment.Generation, DesiredReplicas: desiredReplicas(deployment), UpdatedReplicas: deployment.Status.UpdatedReplicas, AvailableReplicas: deployment.Status.AvailableReplicas, ReadyReplicas: deployment.Status.ReadyReplicas, CompletedAt: completedAt, } w.mu.Lock() // The informer re-delivers every deployment on each resync (~30s). Only an // observable change advances LastSeen, so a quiet rollout keeps its real // last-changed time instead of looking perpetually fresh. if old, exists := w.rollouts[key]; exists && !rolloutChanged(old, rollout) { rollout.LastSeen = old.LastSeen } w.rollouts[key] = rollout if len(w.rollouts) > maxRollouts { w.evictOldestLocked() } w.mu.Unlock() } func (w *Watcher) observeReplicaSet(obj interface{}) { rs, ok := obj.(*appsv1.ReplicaSet) if !ok { return } revision := rs.Annotations["deployment.kubernetes.io/revision"] owner := "" for _, ref := range rs.OwnerReferences { if ref.Kind == "Deployment" && ref.Name != "" { owner = ref.Name } } if revision == "" || owner == "" || rs.CreationTimestamp.IsZero() { return } w.mu.Lock() w.revisions[rs.Namespace+"/"+owner+"/"+revision] = rs.CreationTimestamp.UnixNano() w.mu.Unlock() } func (w *Watcher) revisionCreated(namespace, workload, revision string) int64 { w.mu.RLock() defer w.mu.RUnlock() return w.revisions[namespace+"/"+workload+"/"+revision] } // rolloutChanged reports whether anything observable about the rollout moved. func rolloutChanged(a, b Rollout) bool { return a.Phase != b.Phase || a.Reason != b.Reason || a.Message != b.Message || a.Service != b.Service || a.Release != b.Release || a.Image != b.Image || a.Generation != b.Generation || a.DesiredReplicas != b.DesiredReplicas || a.UpdatedReplicas != b.UpdatedReplicas || a.AvailableReplicas != b.AvailableReplicas || a.ReadyReplicas != b.ReadyReplicas } func (w *Watcher) evictOldestLocked() { var oldest string var ts int64 for key, item := range w.rollouts { if oldest == "" || item.StartedAt < ts { oldest, ts = key, item.StartedAt } } if oldest != "" { delete(w.rollouts, oldest) } } func (w *Watcher) setError(message string) { w.mu.Lock(); defer w.mu.Unlock(); w.err = message } func (w *Watcher) List() []Rollout { w.mu.RLock() defer w.mu.RUnlock() result := make([]Rollout, 0, len(w.rollouts)) for _, item := range w.rollouts { result = append(result, item) } sort.Slice(result, func(i, j int) bool { if result[i].StartedAt != result[j].StartedAt { return result[i].StartedAt > result[j].StartedAt } return result[i].LastSeen > result[j].LastSeen }) return result } func (w *Watcher) Get(id string) (Rollout, bool) { w.mu.RLock() defer w.mu.RUnlock() for _, item := range w.rollouts { if item.ID == id { return item, true } } return Rollout{}, false } func (w *Watcher) Enabled() bool { return w != nil && w.enabled } func (w *Watcher) Error() string { if w == nil { return "kubernetes integration is not configured" } w.mu.RLock() defer w.mu.RUnlock() return w.err } func rolloutID(key string) string { sum := sha256.Sum256([]byte(key)) return hex.EncodeToString(sum[:8]) } func metadataValue(deployment *appsv1.Deployment, keys ...string) string { for _, key := range keys { if value := deployment.Labels[key]; value != "" { return value } if value := deployment.Annotations[key]; value != "" { return value } if value := deployment.Spec.Template.Labels[key]; value != "" { return value } if value := deployment.Spec.Template.Annotations[key]; value != "" { return value } } return "" } func deploymentImage(deployment *appsv1.Deployment) string { if len(deployment.Spec.Template.Spec.Containers) == 0 { return "" } return deployment.Spec.Template.Spec.Containers[0].Image } func desiredReplicas(deployment *appsv1.Deployment) int32 { if deployment.Spec.Replicas == nil { return 1 } return *deployment.Spec.Replicas } func rolloutStatus(deployment *appsv1.Deployment) (phase, reason, message string, completed bool) { phase = "progressing" for _, condition := range deployment.Status.Conditions { switch condition.Type { case appsv1.DeploymentProgressing: reason, message = condition.Reason, condition.Message if condition.Status == corev1.ConditionFalse { return "failed", reason, message, false } case appsv1.DeploymentAvailable: if condition.Status == corev1.ConditionTrue && deployment.Status.UpdatedReplicas == desiredReplicas(deployment) && deployment.Status.AvailableReplicas >= desiredReplicas(deployment) { return "completed", reason, message, true } } } return phase, reason, message, false }