From d35ab43a145236745fb1c979465b67b44cef910e Mon Sep 17 00:00:00 2001 From: morazow Date: Wed, 26 Aug 2026 12:09:44 +0200 Subject: [PATCH 1/2] [helm] Support multi-disk data.dirs for tablet servers Add tablet.storage.volumes, a list of {name, size, storageClass} entries. Each entry renders its own volumeClaimTemplate (or emptyDir when storage is disabled) mounted at /tmp/fluss/, and the chart emits data.dirs with all mount paths in server.yaml so tablet servers balance data across the disks. With the list empty, the chart renders exactly what it does today: one 'data' claim sized by tablet.storage.size and no data.dirs line, so existing releases see no StatefulSet storage change on upgrade. Template-time validation rejects entries missing name or size, duplicate or chart-reserved volume names, and a configurationOverrides data.dirs that would conflict with the generated one. --- helm/templates/_storage.tpl | 79 ++++++ helm/templates/_validate.tpl | 1 + helm/templates/configmap.yaml | 7 + helm/templates/sts-tablet.yaml | 18 +- helm/tests/storage_test.yaml | 248 ++++++++++++++++++ helm/values.yaml | 12 + .../install-deploy/deploying-with-helm.md | 38 +++ 7 files changed, 397 insertions(+), 6 deletions(-) create mode 100644 helm/templates/_storage.tpl create mode 100644 helm/tests/storage_test.yaml diff --git a/helm/templates/_storage.tpl b/helm/templates/_storage.tpl new file mode 100644 index 00000000000..0dba6e89043 --- /dev/null +++ b/helm/templates/_storage.tpl @@ -0,0 +1,79 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Effective tablet storage volumes as a YAML list of {name, size, storageClass}: +tablet.storage.volumes when non-empty, otherwise a single legacy entry built from +tablet.storage.size / tablet.storage.storageClass. Each volume is mounted at +/tmp/fluss/, so the legacy entry keeps the original /tmp/fluss/data path. +Usage: + include "fluss.tablet.storage.volumes" . | fromYamlArray +*/}} +{{- define "fluss.tablet.storage.volumes" -}} +{{- if .Values.tablet.storage.volumes -}} +{{- toYaml .Values.tablet.storage.volumes -}} +{{- else -}} +- name: data + size: {{ .Values.tablet.storage.size }} + storageClass: {{ .Values.tablet.storage.storageClass }} +{{- end -}} +{{- end -}} + +{{/* +Comma-separated data.dirs value for server.yaml, derived from the effective volume +mount paths. +Usage: + include "fluss.tablet.storage.dataDirs" . +*/}} +{{- define "fluss.tablet.storage.dataDirs" -}} +{{- $paths := list -}} +{{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) -}} +{{- $paths = append $paths (printf "/tmp/fluss/%s" .name) -}} +{{- end -}} +{{- join "," $paths -}} +{{- end -}} + +{{/* +Validation errors for tablet.storage.volumes. +Usage: + include "fluss.storage.validateError" . +*/}} +{{- define "fluss.storage.validateError" -}} +{{- $messages := list -}} +{{- $names := list -}} +{{- range .Values.tablet.storage.volumes -}} + {{- if not .name -}} + {{- $messages = append $messages "tablet.storage.volumes: every entry must set name" -}} + {{- else -}} + {{- if has .name $names -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: duplicate volume name %q" .name) -}} + {{- end -}} + {{- if or (eq .name "fluss-conf") (eq .name "sasl-config") (hasPrefix "secret-" .name) -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: volume name %q collides with a chart-managed volume (fluss-conf, sasl-config, secret-*)" .name) -}} + {{- end -}} + {{- if not .size -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: entry %q must set size" .name) -}} + {{- end -}} + {{- $names = append $names .name -}} + {{- end -}} +{{- end -}} +{{- if and .Values.tablet.storage.volumes (hasKey .Values.configurationOverrides "data.dirs") -}} + {{- $messages = append $messages "configurationOverrides must not set data.dirs when tablet.storage.volumes is used: the chart renders data.dirs from the volume list" -}} +{{- end -}} +{{- join "\n" $messages -}} +{{- end -}} diff --git a/helm/templates/_validate.tpl b/helm/templates/_validate.tpl index d79221c97f7..52d5644efa8 100644 --- a/helm/templates/_validate.tpl +++ b/helm/templates/_validate.tpl @@ -41,6 +41,7 @@ Usage: {{- $messages = append $messages (include "fluss.security.validateError" .) -}} {{- $messages = append $messages (include "fluss.metrics.validateError" .) -}} {{- $messages = append $messages (include "fluss.secrets.validateError" .) -}} +{{- $messages = append $messages (include "fluss.storage.validateError" .) -}} {{- $messages = without $messages "" -}} {{- join "\n" $messages -}} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index bfaf0a74c91..0f31d096f68 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -28,6 +28,13 @@ data: {{ $key }}: {{ tpl (printf "%v" $val) $ }} {{- end }} + {{- if .Values.tablet.storage.volumes }} + + ### Storage + + data.dirs: {{ include "fluss.tablet.storage.dataDirs" . }} + {{- end }} + ### Security {{- $internalProtocol := include "fluss.security.listener.protocol" (dict "context" .Values "listener" "internal") | trim -}} diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 5407b2a6a95..8f7af9d98a0 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -164,8 +164,10 @@ spec: volumeMounts: - name: fluss-conf mountPath: /opt/conf - - name: data - mountPath: /tmp/fluss/data + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} + - name: {{ .name }} + mountPath: /tmp/fluss/{{ .name }} + {{- end }} {{- if (include "fluss.security.jaas.required" .) }} - name: sasl-config mountPath: /etc/fluss/conf @@ -182,9 +184,11 @@ spec: configMap: name: fluss-conf-file {{- if not .Values.tablet.storage.enabled }} - - name: data + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} + - name: {{ .name }} emptyDir: {} {{- end }} + {{- end }} {{- if (include "fluss.security.jaas.required" .) }} {{- include "fluss.security.jaas.volumes" . | nindent 8 }} {{- end }} @@ -196,12 +200,14 @@ spec: {{- end }} {{- if .Values.tablet.storage.enabled }} volumeClaimTemplates: + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} - metadata: - name: data + name: {{ .name }} spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .Values.tablet.storage.size }} - storageClassName: {{ .Values.tablet.storage.storageClass }} + storage: {{ .size }} + storageClassName: {{ .storageClass }} + {{- end }} {{- end}} diff --git a/helm/tests/storage_test.yaml b/helm/tests/storage_test.yaml new file mode 100644 index 00000000000..215b39beca6 --- /dev/null +++ b/helm/tests/storage_test.yaml @@ -0,0 +1,248 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: tablet-storage +templates: + - templates/sts-tablet.yaml + - templates/configmap.yaml + - templates/NOTES.txt +tests: + - it: renders a single emptyDir named data by default + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data + mountPath: /tmp/fluss/data + template: templates/sts-tablet.yaml + - notExists: + path: spec.volumeClaimTemplates + template: templates/sts-tablet.yaml + - notMatchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs:' + template: templates/configmap.yaml + + - it: renders the legacy single claim from size and storageClass + set: + tablet.storage.enabled: true + tablet.storage.size: 20Gi + tablet.storage.storageClass: fast + asserts: + - equal: + path: spec.volumeClaimTemplates + value: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi + storageClassName: fast + template: templates/sts-tablet.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + template: templates/sts-tablet.yaml + - notMatchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs:' + template: templates/configmap.yaml + + - it: renders one claim, one mount, and data.dirs per configured volume + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + storageClass: fast + - name: data-1 + size: 200Gi + asserts: + - equal: + path: spec.volumeClaimTemplates + value: + - metadata: + name: data-0 + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Gi + storageClassName: fast + - metadata: + name: data-1 + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 200Gi + storageClassName: null + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data-0 + mountPath: /tmp/fluss/data-0 + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data-1 + mountPath: /tmp/fluss/data-1 + template: templates/sts-tablet.yaml + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/data-0,/tmp/fluss/data-1' + template: templates/configmap.yaml + + - it: renders data.dirs for a single-entry volume list + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: fast-0 + size: 50Gi + asserts: + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/fast-0' + template: templates/configmap.yaml + + - it: renders one emptyDir per volume when storage is disabled + set: + tablet.storage.volumes: + - name: data-0 + size: 100Gi + - name: data-1 + size: 100Gi + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: data-0 + emptyDir: {} + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.volumes + content: + name: data-1 + emptyDir: {} + template: templates/sts-tablet.yaml + - notExists: + path: spec.volumeClaimTemplates + template: templates/sts-tablet.yaml + + - it: keeps extraVolumes and extraVolumeMounts after the storage entries + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + tablet.extraVolumes: + - name: scratch + hostPath: + path: /mnt/scratch + tablet.extraVolumeMounts: + - name: scratch + mountPath: /mnt/scratch + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: scratch + hostPath: + path: /mnt/scratch + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data-0 + mountPath: /tmp/fluss/data-0 + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: scratch + mountPath: /mnt/scratch + template: templates/sts-tablet.yaml + + - it: fails when a volume entry has no size + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: entry \"data-0\" must set size" + template: templates/NOTES.txt + + - it: fails when a volume entry has no name + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: every entry must set name" + template: templates/NOTES.txt + + - it: fails on duplicate volume names + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + - name: data-0 + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: duplicate volume name \"data-0\"" + template: templates/NOTES.txt + + - it: fails on a chart-reserved volume name + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: sasl-config + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: volume name \"sasl-config\" collides with a chart-managed volume (fluss-conf, sasl-config, secret-*)" + template: templates/NOTES.txt + + - it: fails when configurationOverrides also sets data.dirs + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + configurationOverrides: + data.dirs: /elsewhere + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\nconfigurationOverrides must not set data.dirs when tablet.storage.volumes is used: the chart renders data.dirs from the volume list" + template: templates/NOTES.txt diff --git a/helm/values.yaml b/helm/values.yaml index de9c8eb42e0..715a897dad8 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -43,6 +43,18 @@ tablet: enabled: false size: 1Gi storageClass: + # Multi-disk (JBOD): one PVC per entry, mounted at /tmp/fluss/. When + # non-empty, this list takes precedence over size/storageClass above and the + # chart renders `data.dirs` with all mount paths in server.yaml. + # NOTE: volumeClaimTemplates are immutable - enabling this on an existing + # release requires recreating the tablet StatefulSet (see the Helm docs page). + volumes: [] + # - name: data-0 + # size: 100Gi + # storageClass: fast + # - name: data-1 + # size: 100Gi + # storageClass: fast # Readiness probe configuration for rolling upgrade gate. # The probe talks to the LOCAL tablet server (127.0.0.1:client-port) and # asks it to forward a getClusterHealth RPC to the Coordinator over the diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..289d1f6dcb8 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -465,6 +465,10 @@ making rotation fully hands-off. | `tablet.storage.enabled` | Enable persistent volume claims for TabletServer | `false` | | `tablet.storage.size` | Tablet persistent volume size | `1Gi` | | `tablet.storage.storageClass` | Tablet storage class name | `nil` (uses default) | +| `tablet.storage.volumes` | Multi-disk volume list; takes precedence over `size`/`storageClass` | `[]` | +| `tablet.storage.volumes[*].name` | Volume name; mounted at `/tmp/fluss/` | Required | +| `tablet.storage.volumes[*].size` | Persistent volume size for this disk | Required | +| `tablet.storage.volumes[*].storageClass` | Storage class name for this disk | `nil` (uses default) | ### Resource Parameters @@ -711,6 +715,40 @@ tablet: storageClass: fast-ssd ``` +#### Multiple Disks per Tablet Server (JBOD) + +Tablet servers can spread data across multiple local disks via the `data.dirs` +configuration. Declare one volume per disk under `tablet.storage.volumes`; each +entry becomes its own PersistentVolumeClaim mounted at `/tmp/fluss/`, and +the chart renders `data.dirs` with all mount paths in `server.yaml`: + +```yaml +tablet: + storage: + enabled: true + volumes: + - name: data-0 + size: 100Gi + storageClass: fast-ssd + - name: data-1 + size: 100Gi + storageClass: fast-ssd +``` + +When `volumes` is set it takes precedence over the single-disk `size` and +`storageClass` keys, and `configurationOverrides` must not set `data.dirs` +itself. The server ignores the default `data.dir` whenever `data.dirs` is +configured. + +:::warning +Kubernetes forbids changing the `volumeClaimTemplates` of an existing +StatefulSet. To switch an existing release from the single-disk layout to +`volumes` (or to change the volume list), delete the tablet StatefulSet while +keeping its pods and claims (`kubectl delete statefulset tablet-server +--cascade=orphan`) before upgrading, or install a fresh release. Data on the +old single PVC is not migrated to the new volume layout automatically. +::: + Configure remote storage: ```yaml From 8ebd709b268d5804eb03d5dae7b5d5f3d4019b68 Mon Sep 17 00:00:00 2001 From: morazow Date: Wed, 26 Aug 2026 12:48:13 +0200 Subject: [PATCH 2/2] [helm] Harden multi-disk volume validation and rendering Review findings on #4114: - Guard the data.dirs conflict check with 'default dict' so a nulled-out configurationOverrides renders instead of crashing hasKey (matches the chart's existing _metrics.tpl pattern). - Reject volume names that are not lowercase DNS-1123 labels at template time; previously a bad name passed lint and failed only at the API server, and a comma would silently corrupt the joined data.dirs value. - Emit claim size and storageClass via toYaml so string values that look like another YAML type (e.g. storageClass "1234") stay strings; nil storageClass still renders empty, keeping single-disk output identical. - Lock the volumeMounts order in the extraVolumeMounts test with a full equal assert instead of order-insensitive contains. --- helm/templates/_storage.tpl | 5 +++- helm/templates/sts-tablet.yaml | 4 +-- helm/tests/storage_test.yaml | 49 +++++++++++++++++++++++++++------- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/helm/templates/_storage.tpl b/helm/templates/_storage.tpl index 0dba6e89043..39a3df089d6 100644 --- a/helm/templates/_storage.tpl +++ b/helm/templates/_storage.tpl @@ -66,13 +66,16 @@ Usage: {{- if or (eq .name "fluss-conf") (eq .name "sasl-config") (hasPrefix "secret-" .name) -}} {{- $messages = append $messages (printf "tablet.storage.volumes: volume name %q collides with a chart-managed volume (fluss-conf, sasl-config, secret-*)" .name) -}} {{- end -}} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" .name) -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: volume name %q must be a lowercase DNS-1123 label (^[a-z0-9]([-a-z0-9]*[a-z0-9])?$)" .name) -}} + {{- end -}} {{- if not .size -}} {{- $messages = append $messages (printf "tablet.storage.volumes: entry %q must set size" .name) -}} {{- end -}} {{- $names = append $names .name -}} {{- end -}} {{- end -}} -{{- if and .Values.tablet.storage.volumes (hasKey .Values.configurationOverrides "data.dirs") -}} +{{- if and .Values.tablet.storage.volumes (hasKey (.Values.configurationOverrides | default dict) "data.dirs") -}} {{- $messages = append $messages "configurationOverrides must not set data.dirs when tablet.storage.volumes is used: the chart renders data.dirs from the volume list" -}} {{- end -}} {{- join "\n" $messages -}} diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 8f7af9d98a0..d0c6affeeb6 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -207,7 +207,7 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .size }} - storageClassName: {{ .storageClass }} + storage: {{ .size | toYaml }} + storageClassName: {{ if .storageClass }}{{ .storageClass | toYaml }}{{ end }} {{- end }} {{- end}} diff --git a/helm/tests/storage_test.yaml b/helm/tests/storage_test.yaml index 215b39beca6..dac4c979aa6 100644 --- a/helm/tests/storage_test.yaml +++ b/helm/tests/storage_test.yaml @@ -126,11 +126,31 @@ tests: tablet.storage.volumes: - name: fast-0 size: 50Gi + storageClass: "1234" asserts: - matchRegex: path: data["server.yaml"] pattern: 'data\.dirs: /tmp/fluss/fast-0' template: templates/configmap.yaml + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "1234" + template: templates/sts-tablet.yaml + + - it: renders data.dirs when configurationOverrides is nulled out + set: + configurationOverrides: null + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + asserts: + - notFailedTemplate: {} + template: templates/NOTES.txt + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/data-0' + template: templates/configmap.yaml - it: renders one emptyDir per volume when storage is disabled set: @@ -177,17 +197,15 @@ tests: hostPath: path: /mnt/scratch template: templates/sts-tablet.yaml - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: data-0 - mountPath: /tmp/fluss/data-0 - template: templates/sts-tablet.yaml - - contains: + - equal: path: spec.template.spec.containers[0].volumeMounts - content: - name: scratch - mountPath: /mnt/scratch + value: + - name: fluss-conf + mountPath: /opt/conf + - name: data-0 + mountPath: /tmp/fluss/data-0 + - name: scratch + mountPath: /mnt/scratch template: templates/sts-tablet.yaml - it: fails when a volume entry has no size @@ -223,6 +241,17 @@ tests: errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: duplicate volume name \"data-0\"" template: templates/NOTES.txt + - it: fails on a volume name that is not a DNS-1123 label + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: bad_Name + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: volume name \"bad_Name\" must be a lowercase DNS-1123 label (^[a-z0-9]([-a-z0-9]*[a-z0-9])?$)" + template: templates/NOTES.txt + - it: fails on a chart-reserved volume name set: tablet.storage.enabled: true