diff --git a/helm/templates/_storage.tpl b/helm/templates/_storage.tpl new file mode 100644 index 00000000000..39a3df089d6 --- /dev/null +++ b/helm/templates/_storage.tpl @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Effective tablet storage volumes as a YAML list of {name, size, storageClass}: +tablet.storage.volumes when non-empty, otherwise a single legacy entry built from +tablet.storage.size / tablet.storage.storageClass. Each volume is mounted at +/tmp/fluss/, so the legacy entry keeps the original /tmp/fluss/data path. +Usage: + include "fluss.tablet.storage.volumes" . | fromYamlArray +*/}} +{{- define "fluss.tablet.storage.volumes" -}} +{{- if .Values.tablet.storage.volumes -}} +{{- toYaml .Values.tablet.storage.volumes -}} +{{- else -}} +- name: data + size: {{ .Values.tablet.storage.size }} + storageClass: {{ .Values.tablet.storage.storageClass }} +{{- end -}} +{{- end -}} + +{{/* +Comma-separated data.dirs value for server.yaml, derived from the effective volume +mount paths. +Usage: + include "fluss.tablet.storage.dataDirs" . +*/}} +{{- define "fluss.tablet.storage.dataDirs" -}} +{{- $paths := list -}} +{{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) -}} +{{- $paths = append $paths (printf "/tmp/fluss/%s" .name) -}} +{{- end -}} +{{- join "," $paths -}} +{{- end -}} + +{{/* +Validation errors for tablet.storage.volumes. +Usage: + include "fluss.storage.validateError" . +*/}} +{{- define "fluss.storage.validateError" -}} +{{- $messages := list -}} +{{- $names := list -}} +{{- range .Values.tablet.storage.volumes -}} + {{- if not .name -}} + {{- $messages = append $messages "tablet.storage.volumes: every entry must set name" -}} + {{- else -}} + {{- if has .name $names -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: duplicate volume name %q" .name) -}} + {{- end -}} + {{- if or (eq .name "fluss-conf") (eq .name "sasl-config") (hasPrefix "secret-" .name) -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: volume name %q collides with a chart-managed volume (fluss-conf, sasl-config, secret-*)" .name) -}} + {{- end -}} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" .name) -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: volume name %q must be a lowercase DNS-1123 label (^[a-z0-9]([-a-z0-9]*[a-z0-9])?$)" .name) -}} + {{- end -}} + {{- if not .size -}} + {{- $messages = append $messages (printf "tablet.storage.volumes: entry %q must set size" .name) -}} + {{- end -}} + {{- $names = append $names .name -}} + {{- end -}} +{{- end -}} +{{- if and .Values.tablet.storage.volumes (hasKey (.Values.configurationOverrides | default dict) "data.dirs") -}} + {{- $messages = append $messages "configurationOverrides must not set data.dirs when tablet.storage.volumes is used: the chart renders data.dirs from the volume list" -}} +{{- end -}} +{{- join "\n" $messages -}} +{{- end -}} diff --git a/helm/templates/_validate.tpl b/helm/templates/_validate.tpl index d79221c97f7..52d5644efa8 100644 --- a/helm/templates/_validate.tpl +++ b/helm/templates/_validate.tpl @@ -41,6 +41,7 @@ Usage: {{- $messages = append $messages (include "fluss.security.validateError" .) -}} {{- $messages = append $messages (include "fluss.metrics.validateError" .) -}} {{- $messages = append $messages (include "fluss.secrets.validateError" .) -}} +{{- $messages = append $messages (include "fluss.storage.validateError" .) -}} {{- $messages = without $messages "" -}} {{- join "\n" $messages -}} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index bfaf0a74c91..0f31d096f68 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -28,6 +28,13 @@ data: {{ $key }}: {{ tpl (printf "%v" $val) $ }} {{- end }} + {{- if .Values.tablet.storage.volumes }} + + ### Storage + + data.dirs: {{ include "fluss.tablet.storage.dataDirs" . }} + {{- end }} + ### Security {{- $internalProtocol := include "fluss.security.listener.protocol" (dict "context" .Values "listener" "internal") | trim -}} diff --git a/helm/templates/sts-tablet.yaml b/helm/templates/sts-tablet.yaml index 5407b2a6a95..d0c6affeeb6 100644 --- a/helm/templates/sts-tablet.yaml +++ b/helm/templates/sts-tablet.yaml @@ -164,8 +164,10 @@ spec: volumeMounts: - name: fluss-conf mountPath: /opt/conf - - name: data - mountPath: /tmp/fluss/data + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} + - name: {{ .name }} + mountPath: /tmp/fluss/{{ .name }} + {{- end }} {{- if (include "fluss.security.jaas.required" .) }} - name: sasl-config mountPath: /etc/fluss/conf @@ -182,9 +184,11 @@ spec: configMap: name: fluss-conf-file {{- if not .Values.tablet.storage.enabled }} - - name: data + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} + - name: {{ .name }} emptyDir: {} {{- end }} + {{- end }} {{- if (include "fluss.security.jaas.required" .) }} {{- include "fluss.security.jaas.volumes" . | nindent 8 }} {{- end }} @@ -196,12 +200,14 @@ spec: {{- end }} {{- if .Values.tablet.storage.enabled }} volumeClaimTemplates: + {{- range (include "fluss.tablet.storage.volumes" . | fromYamlArray) }} - metadata: - name: data + name: {{ .name }} spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: {{ .Values.tablet.storage.size }} - storageClassName: {{ .Values.tablet.storage.storageClass }} + storage: {{ .size | toYaml }} + storageClassName: {{ if .storageClass }}{{ .storageClass | toYaml }}{{ end }} + {{- end }} {{- end}} diff --git a/helm/tests/storage_test.yaml b/helm/tests/storage_test.yaml new file mode 100644 index 00000000000..dac4c979aa6 --- /dev/null +++ b/helm/tests/storage_test.yaml @@ -0,0 +1,277 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: tablet-storage +templates: + - templates/sts-tablet.yaml + - templates/configmap.yaml + - templates/NOTES.txt +tests: + - it: renders a single emptyDir named data by default + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data + mountPath: /tmp/fluss/data + template: templates/sts-tablet.yaml + - notExists: + path: spec.volumeClaimTemplates + template: templates/sts-tablet.yaml + - notMatchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs:' + template: templates/configmap.yaml + + - it: renders the legacy single claim from size and storageClass + set: + tablet.storage.enabled: true + tablet.storage.size: 20Gi + tablet.storage.storageClass: fast + asserts: + - equal: + path: spec.volumeClaimTemplates + value: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi + storageClassName: fast + template: templates/sts-tablet.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + template: templates/sts-tablet.yaml + - notMatchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs:' + template: templates/configmap.yaml + + - it: renders one claim, one mount, and data.dirs per configured volume + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + storageClass: fast + - name: data-1 + size: 200Gi + asserts: + - equal: + path: spec.volumeClaimTemplates + value: + - metadata: + name: data-0 + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Gi + storageClassName: fast + - metadata: + name: data-1 + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 200Gi + storageClassName: null + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data-0 + mountPath: /tmp/fluss/data-0 + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data-1 + mountPath: /tmp/fluss/data-1 + template: templates/sts-tablet.yaml + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/data-0,/tmp/fluss/data-1' + template: templates/configmap.yaml + + - it: renders data.dirs for a single-entry volume list + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: fast-0 + size: 50Gi + storageClass: "1234" + asserts: + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/fast-0' + template: templates/configmap.yaml + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "1234" + template: templates/sts-tablet.yaml + + - it: renders data.dirs when configurationOverrides is nulled out + set: + configurationOverrides: null + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + asserts: + - notFailedTemplate: {} + template: templates/NOTES.txt + - matchRegex: + path: data["server.yaml"] + pattern: 'data\.dirs: /tmp/fluss/data-0' + template: templates/configmap.yaml + + - it: renders one emptyDir per volume when storage is disabled + set: + tablet.storage.volumes: + - name: data-0 + size: 100Gi + - name: data-1 + size: 100Gi + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: data-0 + emptyDir: {} + template: templates/sts-tablet.yaml + - contains: + path: spec.template.spec.volumes + content: + name: data-1 + emptyDir: {} + template: templates/sts-tablet.yaml + - notExists: + path: spec.volumeClaimTemplates + template: templates/sts-tablet.yaml + + - it: keeps extraVolumes and extraVolumeMounts after the storage entries + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + tablet.extraVolumes: + - name: scratch + hostPath: + path: /mnt/scratch + tablet.extraVolumeMounts: + - name: scratch + mountPath: /mnt/scratch + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: scratch + hostPath: + path: /mnt/scratch + template: templates/sts-tablet.yaml + - equal: + path: spec.template.spec.containers[0].volumeMounts + value: + - name: fluss-conf + mountPath: /opt/conf + - name: data-0 + mountPath: /tmp/fluss/data-0 + - name: scratch + mountPath: /mnt/scratch + template: templates/sts-tablet.yaml + + - it: fails when a volume entry has no size + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: entry \"data-0\" must set size" + template: templates/NOTES.txt + + - it: fails when a volume entry has no name + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: every entry must set name" + template: templates/NOTES.txt + + - it: fails on duplicate volume names + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + - name: data-0 + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: duplicate volume name \"data-0\"" + template: templates/NOTES.txt + + - it: fails on a volume name that is not a DNS-1123 label + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: bad_Name + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: volume name \"bad_Name\" must be a lowercase DNS-1123 label (^[a-z0-9]([-a-z0-9]*[a-z0-9])?$)" + template: templates/NOTES.txt + + - it: fails on a chart-reserved volume name + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: sasl-config + size: 100Gi + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\ntablet.storage.volumes: volume name \"sasl-config\" collides with a chart-managed volume (fluss-conf, sasl-config, secret-*)" + template: templates/NOTES.txt + + - it: fails when configurationOverrides also sets data.dirs + set: + tablet.storage.enabled: true + tablet.storage.volumes: + - name: data-0 + size: 100Gi + configurationOverrides: + data.dirs: /elsewhere + asserts: + - failedTemplate: + errorMessage: "VALUES VALIDATION:\nconfigurationOverrides must not set data.dirs when tablet.storage.volumes is used: the chart renders data.dirs from the volume list" + template: templates/NOTES.txt diff --git a/helm/values.yaml b/helm/values.yaml index de9c8eb42e0..715a897dad8 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -43,6 +43,18 @@ tablet: enabled: false size: 1Gi storageClass: + # Multi-disk (JBOD): one PVC per entry, mounted at /tmp/fluss/. When + # non-empty, this list takes precedence over size/storageClass above and the + # chart renders `data.dirs` with all mount paths in server.yaml. + # NOTE: volumeClaimTemplates are immutable - enabling this on an existing + # release requires recreating the tablet StatefulSet (see the Helm docs page). + volumes: [] + # - name: data-0 + # size: 100Gi + # storageClass: fast + # - name: data-1 + # size: 100Gi + # storageClass: fast # Readiness probe configuration for rolling upgrade gate. # The probe talks to the LOCAL tablet server (127.0.0.1:client-port) and # asks it to forward a getClusterHealth RPC to the Coordinator over the diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..289d1f6dcb8 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -465,6 +465,10 @@ making rotation fully hands-off. | `tablet.storage.enabled` | Enable persistent volume claims for TabletServer | `false` | | `tablet.storage.size` | Tablet persistent volume size | `1Gi` | | `tablet.storage.storageClass` | Tablet storage class name | `nil` (uses default) | +| `tablet.storage.volumes` | Multi-disk volume list; takes precedence over `size`/`storageClass` | `[]` | +| `tablet.storage.volumes[*].name` | Volume name; mounted at `/tmp/fluss/` | Required | +| `tablet.storage.volumes[*].size` | Persistent volume size for this disk | Required | +| `tablet.storage.volumes[*].storageClass` | Storage class name for this disk | `nil` (uses default) | ### Resource Parameters @@ -711,6 +715,40 @@ tablet: storageClass: fast-ssd ``` +#### Multiple Disks per Tablet Server (JBOD) + +Tablet servers can spread data across multiple local disks via the `data.dirs` +configuration. Declare one volume per disk under `tablet.storage.volumes`; each +entry becomes its own PersistentVolumeClaim mounted at `/tmp/fluss/`, and +the chart renders `data.dirs` with all mount paths in `server.yaml`: + +```yaml +tablet: + storage: + enabled: true + volumes: + - name: data-0 + size: 100Gi + storageClass: fast-ssd + - name: data-1 + size: 100Gi + storageClass: fast-ssd +``` + +When `volumes` is set it takes precedence over the single-disk `size` and +`storageClass` keys, and `configurationOverrides` must not set `data.dirs` +itself. The server ignores the default `data.dir` whenever `data.dirs` is +configured. + +:::warning +Kubernetes forbids changing the `volumeClaimTemplates` of an existing +StatefulSet. To switch an existing release from the single-disk layout to +`volumes` (or to change the volume list), delete the tablet StatefulSet while +keeping its pods and claims (`kubectl delete statefulset tablet-server +--cascade=orphan`) before upgrading, or install a fresh release. Data on the +old single PVC is not migrated to the new volume layout automatically. +::: + Configure remote storage: ```yaml