more otel

k8s/clickhouse.yaml

@@ -0,0 +1,134 @@
+# Single-node ClickHouse for the otel-debug-receiver, applied as one file:
+#   kubectl apply -f k8s/clickhouse.yaml
+#
+# A single instance (per request): one StatefulSet replica with a PVC so
+# telemetry survives pod restarts. Resources sized for a small cluster;
+# adjust before production use.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: otel-debug
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clickhouse-auth
+  namespace: monitoring
+type: Opaque
+stringData:
+  # Change these before applying. ClickHouse will honor the values from
+  # the users.d override mounted below, so no post-install SQL is needed.
+  username: otel
+  password: CHANGE_ME
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clickhouse-config
+  namespace: monitoring
+data:
+  users.d-otel.xml: |
+    <clickhouse>
+      <users>
+        <otel>
+          <password from_env="CLICKHOUSE_PASSWORD"/>
+          <networks><ip>::/0</ip></networks>
+          <grants>
+            <grant><query>GRANT ALL ON otel.*</query></grant>
+          </grants>
+        </otel>
+      </users>
+    </clickhouse>
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: clickhouse
+  namespace: monitoring
+spec:
+  serviceName: clickhouse
+  replicas: 1
+  selector:
+    matchLabels:
+      app: clickhouse
+  template:
+    metadata:
+      labels:
+        app: clickhouse
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+      - name: clickhouse
+        image: clickhouse/clickhouse-server:24.8
+        ports:
+        - containerPort: 9000
+          name: native
+        - containerPort: 8123
+          name: http
+        env:
+        - name: CLICKHOUSE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: password
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/clickhouse
+        # Mount the user override as a single file: ConfigMap volumes are
+        # read-only, and the entrypoint needs to write its own
+        # default-user.xml into the users.d directory on first start.
+        - name: user-config
+          mountPath: /etc/clickhouse-server/users.d/otel.xml
+          subPath: users.d-otel.xml
+        readinessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "2Gi"
+            cpu: "1"
+      volumes:
+      - name: user-config
+        configMap:
+          name: clickhouse-config
+          items:
+          - key: users.d-otel.xml
+            path: users.d-otel.xml
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 20Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clickhouse
+  namespace: monitoring
+spec:
+  selector:
+    app: clickhouse
+  ports:
+  - name: native
+    port: 9000
+    targetPort: 9000
+  - name: http
+    port: 8123
+    targetPort: 8123
+  type: ClusterIP

k8s/deployment.yaml

@@ -2,6 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: otel-debug-receiver
+  namespace: monitoring
+  annotations:
+    k8s-helper.io/git-ssh: git@git.k6n.net:mats/go-otel.git
+    k8s-helper.io/git-url: https://git.k6n.net/mats/go-otel
 spec:
   replicas: 1
   selector:
@@ -12,6 +16,7 @@ spec:
       labels:
         app: otel-debug-receiver
     spec:
+      serviceAccountName: otel-debug-receiver
       securityContext:
         runAsNonRoot: true
         runAsUser: 1000
@@ -40,9 +45,77 @@ spec:
           value: "info"
         - name: OTELLOG_PATH
           value: "/logs/otel-agent.log"
+        - name: OTEL_SOURCEMAP_DIR
+          value: "/sourcemaps"
+        - name: OTEL_OIDC_ISSUER
+          value: "https://users.k6n.net"
+        - name: OTEL_OIDC_CLIENT_ID
+          value: "otel-debug"
+        # Service credential for non-interactive debugging agents. It grants
+        # access to protected /api debug endpoints, not OTLP ingest.
+        - name: OTEL_AGENT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: otel-agent-auth
+              key: token
+              # The debug API must not start without its service credential.
+              # Leaving this optional makes the pod appear healthy while every
+              # protected API request can only return 401.
+              optional: false
+        # ClickHouse storage: enables the aggregate stats endpoints
+        # (/api/stats/*). Falls back to the file store if unreachable.
+        - name: OTEL_CLICKHOUSE_ADDR
+          value: "clickhouse.monitoring.svc.cluster.local:9000"
+        - name: OTEL_CLICKHOUSE_DB
+          value: "otel"
+        - name: OTEL_CLICKHOUSE_USER
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: username
+              optional: true
+        - name: OTEL_CLICKHOUSE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: password
+              optional: true
+        # Connection pool tuning to prevent "acquire conn timeout" under ingest bursts
+        - name: OTEL_CLICKHOUSE_MAX_OPEN_CONNS
+          value: "10"
+        - name: OTEL_CLICKHOUSE_MAX_IDLE_CONNS
+          value: "5"
+        - name: OTEL_CLICKHOUSE_DIAL_TIMEOUT
+          value: "30s"
+        - name: OTEL_CLICKHOUSE_CONN_MAX_LIFETIME
+          value: "10m"
+        - name: OTEL_STORE_WORKERS
+          value: "4"
+        # Optional rollout observation. The watcher is read-only and uses the
+        # service account below. Leave Prometheus unset when unavailable.
+        - name: OTEL_KUBERNETES_NAMESPACE
+          value: ""
+        # - name: OTEL_PROMETHEUS_URL
+        #   value: "http://prometheus.monitoring.svc.cluster.local:9090"
+        # Private token for an ingress-side synchronizer. If absent, the
+        # promoted-IP endpoint stays disabled.
+        - name: OTEL_HONEYPOT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: honeypot-auth
+              key: token
+              optional: true
+        - name: OTEL_HONEYPOT_THRESHOLD
+          value: "3"
+        - name: OTEL_HONEYPOT_WINDOW
+          value: "15m"
+        - name: OTEL_HONEYPOT_TTL
+          value: "1h"
         volumeMounts:
         - name: log-dir
           mountPath: /logs
+        - name: sourcemap-dir
+          mountPath: /sourcemaps
         livenessProbe:
           httpGet:
             path: /health
@@ -65,3 +138,39 @@ spec:
       volumes:
       - name: log-dir
         emptyDir: {}
+      - name: sourcemap-dir
+        emptyDir: {}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: otel-debug-receiver
+  namespace: monitoring
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otel-debug-receiver-rollout-reader
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments", "replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["pods", "events"]
+  verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: otel-debug-receiver-rollout-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: otel-debug-receiver-rollout-reader
+subjects:
+- kind: ServiceAccount
+  name: otel-debug-receiver
+  namespace: monitoring

k8s/ingress.yaml

@@ -1,11 +1,99 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
+metadata:
+  name: otel-debug-ingress-public
+  namespace: monitoring
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - otel.k6n.net
+      secretName: otel-k6n-tls
+  rules:
+    - host: otel.k6n.net
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /v1
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /health
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          # The handler has its own static token authentication. This route
+          # lets an in-cluster ingress synchronizer fetch promoted IPs without
+          # requiring a browser OIDC session.
+          - path: /internal/honeypot
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /ws
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /agents_example.md
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /instrumentation.md
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /static
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
 metadata:
   name: otel-debug-ingress
+  namespace: monitoring
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
+    # OIDC protection via the platform auth service (users.k6n.net):
+    # /oauth2/auth resolves the session cookie / bearer token and answers
+    # 200 (allow), 401 (challenge -> browser_redirect to /login) or 403.
+    # otel.k6n.net is registered as a realm host in the auth service's
+    # realms.json. Public API and agent endpoints are routed via otel-debug-ingress-public.
+    nginx.ingress.kubernetes.io/auth-url: https://users.k6n.net/oauth2/auth
+    nginx.ingress.kubernetes.io/auth-signin: https://users.k6n.net/login?rd=$scheme://$host$escaped_request_uri
+    nginx.ingress.kubernetes.io/auth-response-headers: X-Auth-User
+    nginx.ingress.kubernetes.io/auth-cache-duration: 200 60s, 401 5s, 403 5s
+    nginx.ingress.kubernetes.io/auth-proxy-set-headers: monitoring/auth-subrequest-headers
 spec:
-  ingressClassName: nginx # Assuming nginx ingress controller
+  ingressClassName: nginx
   tls:
     - hosts:
         - otel.k6n.net
@@ -21,3 +109,13 @@ spec:
                 name: otel-debug-service
                 port:
                   number: 80
+---
+# Header map for the auth subrequest (same pattern as home/auth-subrequest-headers).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: auth-subrequest-headers
+  namespace: monitoring
+data:
+  X-Original-Method: $request_method
+  X-Original-URL: $scheme://$host$request_uri