go-otel/k8s/deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-debug-receiver
  namespace: monitoring
  annotations:
    k8s-helper.io/git-ssh: git@git.k6n.net:mats/go-otel.git
    k8s-helper.io/git-url: https://git.k6n.net/mats/go-otel
spec:
  replicas: 1
  selector:
    matchLabels:
      app: otel-debug-receiver
  template:
    metadata:
      labels:
        app: otel-debug-receiver
    spec:
      serviceAccountName: otel-debug-receiver
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
      containers:
      - name: otel-debug
        image: matst80/otel-debug:latest
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
        ports:
        - containerPort: 8080
          name: http
        - containerPort: 4317
          name: grpc
        - containerPort: 4318
          name: otlp-http
        - containerPort: 6060
          name: pprof
        env:
        - name: GRPC_GO_LOG_SEVERITY_LEVEL
          value: "info"
        - name: OTELLOG_PATH
          value: "/logs/otel-agent.log"
        - name: OTEL_SOURCEMAP_DIR
          value: "/sourcemaps"
        # Shared key required to POST /api/sourcemaps (X-Sourcemap-Key header);
        # uploads are refused while it is unset.
        - name: OTEL_SOURCEMAP_KEY
          value: "6901e62e4468631f27f1dc85fd4231f35b5e591c61f5bcfa"
        # AbuseIPDB API key for authenticated IP checks/reports from the UI.
        # Create out of band: kubectl create secret generic abuseipdb-key -n monitoring --from-literal=key=...
        - name: ABUSE_KEY
          valueFrom:
            secretKeyRef:
              name: abuseipdb-key
              key: key
              optional: true
        - name: OTEL_OIDC_ISSUER
          value: "https://users.k6n.net"
        - name: OTEL_OIDC_CLIENT_ID
          value: "otel-debug"
        # Also accept tokens whose aud carries this service's URL (device-grant
        # dev tokens from the dev-console client, see k8s/auth-dev.yaml).
        - name: OTEL_OIDC_AUDIENCE
          value: "https://otel.k6n.net"
        # Service credential for non-interactive debugging agents. It grants
        # access to protected /api debug endpoints, not OTLP ingest.
        - name: OTEL_AGENT_TOKEN
          valueFrom:
            secretKeyRef:
              name: otel-agent-auth
              key: token
              # The debug API must not start without its service credential.
              # Leaving this optional makes the pod appear healthy while every
              # protected API request can only return 401.
              optional: false
        # ClickHouse storage: enables the aggregate stats endpoints
        # (/api/stats/*). Falls back to the file store if unreachable.
        - name: OTEL_CLICKHOUSE_ADDR
          value: "clickhouse.monitoring.svc.cluster.local:9000"
        - name: OTEL_CLICKHOUSE_DB
          value: "otel"
        - name: OTEL_CLICKHOUSE_USER
          valueFrom:
            secretKeyRef:
              name: clickhouse-auth
              key: username
              optional: true
        - name: OTEL_CLICKHOUSE_PASSWORD
          valueFrom:
            secretKeyRef:
              name: clickhouse-auth
              key: password
              optional: true
        # Connection pool tuning to prevent "acquire conn timeout" under ingest bursts
        - name: OTEL_CLICKHOUSE_MAX_OPEN_CONNS
          value: "10"
        - name: OTEL_CLICKHOUSE_MAX_IDLE_CONNS
          value: "5"
        - name: OTEL_CLICKHOUSE_DIAL_TIMEOUT
          value: "30s"
        - name: OTEL_CLICKHOUSE_CONN_MAX_LIFETIME
          value: "10m"
        # Ingress blocker: promotes scanner IPs into ingress-nginx block-cidrs.
        # Dry-run first: it logs what it would block and exposes it at
        # /api/blocker/status. Set OTEL_BLOCKER_DRY_RUN=false to enforce, and
        # put your own public IP(s) in OTEL_BLOCKER_ALLOW before doing so.
        - name: OTEL_BLOCKER_ENABLED
          value: "true"
        - name: OTEL_BLOCKER_DRY_RUN
          value: "true"
        - name: OTEL_BLOCKER_ALLOW
          value: ""
        # Spike alerts to Telegram. The secret is created out of band (never
        # commit values): kubectl create secret generic otel-alerts -n monitoring
        #   --from-literal=telegram-token=... --from-literal=telegram-chat-id=...
        # Set OTEL_ALERT_DRY_RUN=true to evaluate and record without sending.
        - name: OTEL_ALERT_TELEGRAM_TOKEN
          valueFrom:
            secretKeyRef:
              name: otel-alerts
              key: telegram-token
              optional: true
        - name: OTEL_ALERT_TELEGRAM_CHAT_ID
          valueFrom:
            secretKeyRef:
              name: otel-alerts
              key: telegram-chat-id
              optional: true
        - name: OTEL_ALERT_BASE_URL
          value: "https://otel.k6n.net"
        # Optional Always-On Display session API.
        - name: OTEL_ALERT_AOD_URL
          value: "http://10.10.11.117:35851"
        - name: OTEL_STORE_WORKERS
          value: "4"
        # Optional rollout observation. The watcher is read-only and uses the
        # service account below. Leave Prometheus unset when unavailable.
        - name: OTEL_KUBERNETES_NAMESPACE
          value: ""
        # Enable the explicitly requested, rate-limited Nmap service probe.
        # This is kept opt-in because it generates active traffic to external IPs.
        - name: OTEL_IP_INVESTIGATION_NMAP
          value: "true"
        # Whole-scan budget; scans run in the background and are stored.
        - name: OTEL_IP_INVESTIGATION_NMAP_TIMEOUT
          value: "5m"
        # - name: OTEL_PROMETHEUS_URL
        #   value: "http://prometheus.monitoring.svc.cluster.local:9090"
        # Private token for an ingress-side synchronizer. If absent, the
        # promoted-IP endpoint stays disabled.
        - name: OTEL_HONEYPOT_TOKEN
          valueFrom:
            secretKeyRef:
              name: honeypot-auth
              key: token
              optional: true
        - name: OTEL_HONEYPOT_THRESHOLD
          value: "3"
        - name: OTEL_HONEYPOT_WINDOW
          value: "15m"
        - name: OTEL_HONEYPOT_TTL
          value: "1h"
        - name: OTEL_NEEDLE_ADDR
          value: "http://127.0.0.1:5005"
        # ~80% of the memory limit so the GC works harder before an OOMKill.
        - name: GOMEMLIMIT
          value: "800MiB"
        volumeMounts:
        - name: log-dir
          mountPath: /logs
        - name: sourcemap-dir
          mountPath: /sourcemaps
        livenessProbe:
          httpGet:
            path: /health
            port: http
          initialDelaySeconds: 5
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: http
          initialDelaySeconds: 5
          periodSeconds: 10
        resources:
          requests:
            memory: "512Mi"
            cpu: "100m"
          limits:
            memory: "1Gi"
            cpu: "500m"
      - name: needle-sidecar
        image: matst80/needle-sidecar:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 5005
          name: needle-http
        env:
        - name: HOME
          value: "/app"
        - name: NEEDLE_PORT
          value: "5005"
        - name: NEEDLE_HOST
          value: "0.0.0.0"
        - name: NEEDLE_TELEMETRY
          value: "0"
        livenessProbe:
          httpGet:
            path: /health
            port: needle-http
          initialDelaySeconds: 5
          periodSeconds: 15
        readinessProbe:
          httpGet:
            path: /health
            port: needle-http
          initialDelaySeconds: 3
          periodSeconds: 10
        resources:
          requests:
            memory: "512Mi"
            cpu: "200m"
          limits:
            memory: "2Gi"
            cpu: "4000m"
      volumes:
      - name: log-dir
        emptyDir: {}
      - name: sourcemap-dir
        persistentVolumeClaim:
          claimName: otel-sourcemaps

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: otel-sourcemaps
  namespace: monitoring
spec:
  accessModes: ["ReadWriteOnce"]
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-debug-receiver
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-debug-receiver-rollout-reader
rules:
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["pods", "events", "services"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get"]
- apiGroups: ["networking.k8s.io"]
  resources: ["ingresses"]
  verbs: ["get", "list", "watch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-debug-receiver-rollout-reader
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: otel-debug-receiver-rollout-reader
subjects:
- kind: ServiceAccount
  name: otel-debug-receiver
  namespace: monitoring

---
# The blocker may read and update exactly one ConfigMap: the ingress-nginx
# controller's, where `block-cidrs` lives. Nothing else in that namespace.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: otel-debug-receiver-ip-blocker
  namespace: ingress-nginx
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  resourceNames: ["ingress-nginx-controller"]
  verbs: ["get", "update"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: otel-debug-receiver-ip-blocker
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: otel-debug-receiver-ip-blocker
subjects:
- kind: ServiceAccount
  name: otel-debug-receiver
  namespace: monitoring