mreo ui

k8s/clickhouse.yaml

@@ -1,9 +1,12 @@
 # Single-node ClickHouse for the otel-debug-receiver, applied as one file:
 #   kubectl apply -f k8s/clickhouse.yaml
 #
-# A single instance (per request): one StatefulSet replica with a PVC so
-# telemetry survives pod restarts. Resources sized for a small cluster;
-# adjust before production use.
+# A single instance (per request): one StatefulSet replica. Data lives on
+# serber's local disk at /srv/monitoring-db (hostPath), so the pod is pinned to
+# that node. It used to be an nfs-client PVC on a spinning-disk NAS, where
+# synchronous writes cost ~120 ms and every query paid that latency; see
+# k8s/migrate/clickhouse-nfs-to-local.yaml for the one-off data move.
+# Resources sized for a small cluster; adjust before production use.
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -56,8 +59,21 @@ spec:
       labels:
         app: clickhouse
     spec:
+      nodeSelector:
+        kubernetes.io/hostname: serber
       securityContext:
         fsGroup: 101
+      initContainers:
+      # hostPath directories are created root-owned and fsGroup does not apply
+      # to them; ClickHouse runs as uid/gid 101.
+      - name: fix-data-perms
+        image: busybox:1.36
+        command: ["sh", "-c", "chown -R 101:101 /var/lib/clickhouse"]
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/clickhouse
       containers:
       - name: clickhouse
         image: clickhouse/clickhouse-server:24.8
@@ -101,20 +117,16 @@ spec:
             memory: "2Gi"
             cpu: "1"
       volumes:
+      - name: data
+        hostPath:
+          path: /srv/monitoring-db
+          type: DirectoryOrCreate
       - name: user-config
         configMap:
           name: clickhouse-config
           items:
           - key: users.d-otel.xml
             path: users.d-otel.xml
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: ["ReadWriteOnce"]
-      resources:
-        requests:
-          storage: 20Gi
 ---
 apiVersion: v1
 kind: Service

k8s/migrate/clickhouse-nfs-to-local.yaml

@@ -0,0 +1,57 @@
+# One-off: copy ClickHouse data from the old NFS PVC to serber:/srv/monitoring-db.
+# NOT applied by `kubectl apply -f k8s/` (subdirectory). Run it while the
+# StatefulSet is scaled to 0 so the copy is consistent:
+#
+#   kubectl scale sts clickhouse -n monitoring --replicas=0
+#   kubectl apply -f k8s/migrate/clickhouse-nfs-to-local.yaml
+#   kubectl wait --for=condition=complete job/clickhouse-nfs-to-local -n monitoring --timeout=30m
+#   kubectl delete sts clickhouse -n monitoring          # PVC is retained (rollback copy)
+#   kubectl apply -f k8s/clickhouse.yaml                 # new hostPath StatefulSet
+#   kubectl delete job clickhouse-nfs-to-local -n monitoring
+#
+# The old PVC (data-clickhouse-0) is left untouched until you delete it.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: clickhouse-nfs-to-local
+  namespace: monitoring
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: serber
+      containers:
+      - name: copy
+        image: busybox:1.36
+        securityContext:
+          runAsUser: 0
+        command:
+        - sh
+        - -c
+        - |
+          set -eu
+          if [ -n "$(ls -A /new 2>/dev/null)" ]; then
+            echo "refusing to copy: /new (serber:/srv/monitoring-db) is not empty"; exit 1
+          fi
+          du -sh /old
+          cp -a /old/. /new/
+          chown -R 101:101 /new
+          echo "old: $(find /old -type f | wc -l) files, new: $(find /new -type f | wc -l) files"
+          [ "$(find /old -type f | wc -l)" = "$(find /new -type f | wc -l)" ]
+        volumeMounts:
+        - name: old
+          mountPath: /old
+          readOnly: true
+        - name: new
+          mountPath: /new
+      volumes:
+      - name: old
+        persistentVolumeClaim:
+          claimName: data-clickhouse-0
+          readOnly: true
+      - name: new
+        hostPath:
+          path: /srv/monitoring-db
+          type: DirectoryOrCreate