more otel
This commit is contained in:
mats committed
1 parent
9620e3bfa6
commit
21f04d696b
88 files changed
+23419
-192
No files matched your search
@@ -0,0 +1,239 @@
|
||||
// Package kubernetes observes read-only workload state for rollout correlation.
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
const maxRollouts = 500
|
||||
|
||||
type Watcher struct {
|
||||
mu sync.RWMutex
|
||||
rollouts map[string]Rollout
|
||||
enabled bool
|
||||
err string
|
||||
}
|
||||
|
||||
// NewFromEnv enables observation in-cluster, or with KUBECONFIG locally. It is
|
||||
// disabled by default outside Kubernetes so local development needs no kubeconfig.
|
||||
func NewFromEnv(ctx context.Context) (*Watcher, error) {
|
||||
if strings.EqualFold(os.Getenv("OTEL_KUBERNETES_ENABLED"), "false") {
|
||||
return nil, nil
|
||||
}
|
||||
config, err := kubeConfig()
|
||||
if err != nil {
|
||||
if os.Getenv("OTEL_KUBERNETES_ENABLED") == "true" {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
client, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create kubernetes client: %w", err)
|
||||
}
|
||||
|
||||
w := &Watcher{rollouts: make(map[string]Rollout), enabled: true}
|
||||
namespace := os.Getenv("OTEL_KUBERNETES_NAMESPACE")
|
||||
selector := os.Getenv("OTEL_KUBERNETES_LABEL_SELECTOR")
|
||||
factory := informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
|
||||
informers.WithTweakListOptions(func(options *metav1.ListOptions) {
|
||||
options.LabelSelector = selector
|
||||
}),
|
||||
)
|
||||
if namespace != "" {
|
||||
factory = informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
|
||||
informers.WithNamespace(namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) {
|
||||
options.LabelSelector = selector
|
||||
}))
|
||||
}
|
||||
informer := factory.Apps().V1().Deployments().Informer()
|
||||
_, _ = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) { w.observe(obj) },
|
||||
UpdateFunc: func(_, obj interface{}) { w.observe(obj) },
|
||||
})
|
||||
go func() {
|
||||
factory.Start(ctx.Done())
|
||||
if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
|
||||
w.setError("deployment informer did not sync")
|
||||
return
|
||||
}
|
||||
<-ctx.Done()
|
||||
}()
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func kubeConfig() (*rest.Config, error) {
|
||||
if config, err := rest.InClusterConfig(); err == nil {
|
||||
return config, nil
|
||||
}
|
||||
kubeconfig := os.Getenv("KUBECONFIG")
|
||||
if kubeconfig == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kubeconfig = home + string(os.PathSeparator) + ".kube" + string(os.PathSeparator) + "config"
|
||||
}
|
||||
return clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
}
|
||||
|
||||
func (w *Watcher) observe(obj interface{}) {
|
||||
deployment, ok := obj.(*appsv1.Deployment)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
now := time.Now().UnixNano()
|
||||
revision := deployment.Annotations["deployment.kubernetes.io/revision"]
|
||||
if revision == "" {
|
||||
revision = strconv.FormatInt(deployment.Generation, 10)
|
||||
}
|
||||
key := deployment.Namespace + "/" + deployment.Name + "/" + revision
|
||||
phase, reason, message, completed := rolloutStatus(deployment)
|
||||
started := now
|
||||
for _, condition := range deployment.Status.Conditions {
|
||||
if condition.Type == appsv1.DeploymentProgressing && !condition.LastUpdateTime.IsZero() {
|
||||
started = condition.LastUpdateTime.UnixNano()
|
||||
}
|
||||
if condition.Type == appsv1.DeploymentAvailable && !condition.LastUpdateTime.IsZero() && completed {
|
||||
completed = condition.LastUpdateTime.UnixNano() != 0
|
||||
}
|
||||
}
|
||||
if old, exists := w.rollouts[key]; exists {
|
||||
started = old.StartedAt
|
||||
if old.CompletedAt != 0 {
|
||||
completed = true
|
||||
}
|
||||
}
|
||||
rollout := Rollout{
|
||||
ID: rolloutID(key), Namespace: deployment.Namespace, Workload: deployment.Name,
|
||||
Service: metadataValue(deployment, "app.kubernetes.io/name", "app", "service.name"),
|
||||
Release: metadataValue(deployment, "otel.k6n.net/release", "app.kubernetes.io/version", "service.release"),
|
||||
Revision: revision, Image: deploymentImage(deployment), Phase: phase, Reason: reason,
|
||||
Message: message, StartedAt: started, LastSeen: now, Generation: deployment.Generation,
|
||||
DesiredReplicas: desiredReplicas(deployment), UpdatedReplicas: deployment.Status.UpdatedReplicas,
|
||||
AvailableReplicas: deployment.Status.AvailableReplicas, ReadyReplicas: deployment.Status.ReadyReplicas,
|
||||
}
|
||||
if completed {
|
||||
rollout.CompletedAt = now
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.rollouts[key] = rollout
|
||||
if len(w.rollouts) > maxRollouts {
|
||||
w.evictOldestLocked()
|
||||
}
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func (w *Watcher) evictOldestLocked() {
|
||||
var oldest string
|
||||
var ts int64
|
||||
for key, item := range w.rollouts {
|
||||
if oldest == "" || item.LastSeen < ts {
|
||||
oldest, ts = key, item.LastSeen
|
||||
}
|
||||
}
|
||||
if oldest != "" {
|
||||
delete(w.rollouts, oldest)
|
||||
}
|
||||
}
|
||||
func (w *Watcher) setError(message string) { w.mu.Lock(); defer w.mu.Unlock(); w.err = message }
|
||||
|
||||
func (w *Watcher) List() []Rollout {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
result := make([]Rollout, 0, len(w.rollouts))
|
||||
for _, item := range w.rollouts {
|
||||
result = append(result, item)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].LastSeen > result[j].LastSeen })
|
||||
return result
|
||||
}
|
||||
func (w *Watcher) Get(id string) (Rollout, bool) {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
for _, item := range w.rollouts {
|
||||
if item.ID == id {
|
||||
return item, true
|
||||
}
|
||||
}
|
||||
return Rollout{}, false
|
||||
}
|
||||
func (w *Watcher) Enabled() bool { return w != nil && w.enabled }
|
||||
func (w *Watcher) Error() string {
|
||||
if w == nil {
|
||||
return "kubernetes integration is not configured"
|
||||
}
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
return w.err
|
||||
}
|
||||
|
||||
func rolloutID(key string) string {
|
||||
sum := sha256.Sum256([]byte(key))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
func metadataValue(deployment *appsv1.Deployment, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value := deployment.Labels[key]; value != "" {
|
||||
return value
|
||||
}
|
||||
if value := deployment.Annotations[key]; value != "" {
|
||||
return value
|
||||
}
|
||||
if value := deployment.Spec.Template.Labels[key]; value != "" {
|
||||
return value
|
||||
}
|
||||
if value := deployment.Spec.Template.Annotations[key]; value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func deploymentImage(deployment *appsv1.Deployment) string {
|
||||
if len(deployment.Spec.Template.Spec.Containers) == 0 {
|
||||
return ""
|
||||
}
|
||||
return deployment.Spec.Template.Spec.Containers[0].Image
|
||||
}
|
||||
func desiredReplicas(deployment *appsv1.Deployment) int32 {
|
||||
if deployment.Spec.Replicas == nil {
|
||||
return 1
|
||||
}
|
||||
return *deployment.Spec.Replicas
|
||||
}
|
||||
|
||||
func rolloutStatus(deployment *appsv1.Deployment) (phase, reason, message string, completed bool) {
|
||||
phase = "progressing"
|
||||
for _, condition := range deployment.Status.Conditions {
|
||||
switch condition.Type {
|
||||
case appsv1.DeploymentProgressing:
|
||||
reason, message = condition.Reason, condition.Message
|
||||
if condition.Status == corev1.ConditionFalse {
|
||||
return "failed", reason, message, false
|
||||
}
|
||||
case appsv1.DeploymentAvailable:
|
||||
if condition.Status == corev1.ConditionTrue && deployment.Status.UpdatedReplicas == desiredReplicas(deployment) && deployment.Status.AvailableReplicas >= desiredReplicas(deployment) {
|
||||
return "completed", reason, message, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return phase, reason, message, false
|
||||
}
|
||||
Reference in new issue
Block a user