From 6a3c968e11d628d6977dbd5a74e7db485375d0de Mon Sep 17 00:00:00 2001 From: xliuqq Date: Mon, 18 May 2026 19:19:00 +0800 Subject: [PATCH 1/7] support tiered level store for worker/client Signed-off-by: xliuqq --- api/v1alpha1/cacheruntime_types.go | 136 +++--- api/v1alpha1/openapi_generated.go | 198 +++++---- api/v1alpha1/zz_generated.deepcopy.go | 127 +++--- .../crds/data.fluid.io_cacheruntimes.yaml | 356 ++++----------- .../bases/data.fluid.io_cacheruntimes.yaml | 356 ++++----------- .../zh/userguide/cache_runtime_tieredstore.md | 280 ++++++++++++ pkg/common/cacheruntime.go | 32 ++ pkg/ddc/cache/engine/cm.go | 55 +++ pkg/ddc/cache/engine/transform_client.go | 6 + pkg/ddc/cache/engine/transform_client_test.go | 20 +- .../cache/engine/transform_tiered_store.go | 215 ++++++++++ .../engine/transform_tiered_store_test.go | 406 ++++++++++++++++++ pkg/ddc/cache/engine/transform_worker.go | 8 +- pkg/ddc/cache/engine/util.go | 17 + test/gha-e2e/curvine/cacheruntime.yaml | 9 +- test/gha-e2e/curvine/cacheruntimeclass.yaml | 7 +- 16 files changed, 1442 insertions(+), 786 deletions(-) create mode 100644 docs/zh/userguide/cache_runtime_tieredstore.md create mode 100644 pkg/ddc/cache/engine/transform_tiered_store.go create mode 100644 pkg/ddc/cache/engine/transform_tiered_store_test.go diff --git a/api/v1alpha1/cacheruntime_types.go b/api/v1alpha1/cacheruntime_types.go index 627a089b795..524a497e1fd 100644 --- a/api/v1alpha1/cacheruntime_types.go +++ b/api/v1alpha1/cacheruntime_types.go @@ -216,97 +216,89 @@ type RuntimeTieredStore struct { Levels []RuntimeTieredStoreLevel `json:"levels,omitempty"` } -// RuntimeTieredStoreLevel describes the configuration for a single tier in the tiered storage -type RuntimeTieredStoreLevel struct { - // Medium describes the storage medium type for this tier. - // Supported types include process memory and various volume types. - // +optional - Medium MediumSource `json:"medium,omitempty"` +// ProcessMemoryMediumSource describes process memory as a storage medium. +// Cache data will be stored in the process's memory space. +type ProcessMemoryMediumSource struct { + // Quota specifies the amount of memory to allocate for caching. + // This value will be added to the container's resource requests and limits. + // Example: "4Gi" allocates 4GiB memory for cache. + // +kubebuilder:validation:Required + Quota resource.Quantity `json:"quota"` +} - // Path is a list of file paths to be used for the cache tier. - // Multiple paths can be specified to distribute cache across different mount points. - // For example: ["/mnt/cache1", "/mnt/cache2"]. +// HostPathMediumSource describes hostPath volumes as a storage medium. +// Multiple paths can be configured to distribute cache across different disks. +type HostPathMediumSource struct { + // Paths is a list of file paths on the host machine to be used for caching. + // Multiple paths allow distributing cache across different mount points or disks. + // Example: ["/mnt/cache1", "/mnt/cache2"] // +kubebuilder:validation:MinItems=1 - // +optional - Path []string `json:"path,omitempty"` + // +kubebuilder:validation:Required + Paths []string `json:"paths"` - // Quota is a list of storage quotas for each path in the tier. - // The length of Quota should match the length of Path. - // Each quota corresponds to the path at the same index. - // For example: ["100Gi", "50Gi"] allocates 100GiB to the first path and 50GiB to the second path. - // +optional - Quota []resource.Quantity `json:"quota,omitempty"` + // Quotas is a list of storage quotas corresponding to each path. + // The length of Quotas must match the length of Paths. + // Each quota defines the maximum cache size for the corresponding path. + // Example: ["100Gi", "50Gi"] allocates 100GiB to the first path and 50GiB to the second. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:Required + Quotas []resource.Quantity `json:"quotas"` - // High is the ratio of high watermark of the tier (e.g., "0.9"). - // When cache usage exceeds this ratio, eviction will be triggered. + // Type specifies the type of hostPath volume. + // Defaults to empty string (no validation). + // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath // +optional - High string `json:"high,omitempty"` + Type *corev1.HostPathType `json:"type,omitempty"` +} - // Low is the ratio of low watermark of the tier (e.g., "0.7"). - // Eviction will continue until cache usage falls below this ratio. +// EmptyDirMediumSource describes emptyDir volume as a storage medium. +// Can be backed by node's default storage or memory (tmpfs). +type EmptyDirMediumSource struct { + // Quota specifies the maximum storage capacity for the emptyDir volume. + // For Memory medium, this limit is also constrained by the sum of container memory limits. + // The actual limit = min(Quota, sum of container memory limits). + // Example: "100Gi" limits the emptyDir to 100GiB. + // +kubebuilder:validation:Required + Quota resource.Quantity `json:"quota"` + + // medium represents what type of storage medium should back this directory. + // The default is "" which means to use the node's default medium. + // Must be an empty string (default) or Memory. + // More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir // +optional - Low string `json:"low,omitempty"` + Medium corev1.StorageMedium `json:"medium,omitempty"` } -// MediumSource describes the storage medium type for tiered store. -// Only one of its members may be specified. -type MediumSource struct { - // ProcessMemory indicates that process memory should be used as the storage medium. - // The cache will be stored in the process's memory space. +// RuntimeTieredStoreLevel describes the configuration for a single tier in the tiered storage. +// Each tier can use different storage media (e.g., memory, SSD, HDD). +// Only one of ProcessMemory, EmptyDir, or HostPath should be specified. +type RuntimeTieredStoreLevel struct { + // ProcessMemory indicates that process memory should be used as one storage medium. + // When specified, cache data will be stored in the process's memory space, + // and the quota will be added to the container's resource requests and limits. // +optional ProcessMemory *ProcessMemoryMediumSource `json:"processMemory,omitempty"` - // Volume indicates that a Kubernetes volume should be used as the storage medium. - // Supported volume types include hostPath, emptyDir, and ephemeral volumes. + // EmptyDir indicates that an emptyDir volume should be used as the storage medium. // +optional - Volume *VolumeMediumSource `json:"volume,omitempty"` -} - -//tieredStore: -// levels: -// - quota: 8Gi # quota will add to component.container.resource.request/limit.memory -// high: "0.99" -// low: "0.99" -// mediumSource: -// processMemory: {} - -// ProcessMemoryMediumSource describes process memory as a storage medium. -// When specified, cache data will be stored in the process's memory space, -// and the quota will be added to the container's resource requests and limits. -type ProcessMemoryMediumSource struct { -} + EmptyDir *EmptyDirMediumSource `json:"emptyDir,omitempty"` -//tieredStore: -// levels: -// - quota: 8Gi -// high: "0.99" -// low: "0.99" -// path: /dev/shm -// mediumSource: -// emptyDir:{} -// # Or one of the following: -// # ephemeral: -// # volumeClaimTemplate:{} -// # hostPath:{} - -// VolumeMediumSource describes a Kubernetes volume as a storage medium. -// Only one of its members may be specified. -type VolumeMediumSource struct { - // HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. - // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + // HostPath indicates that one or more hostPath volumes should be used as a storage medium. // +optional - HostPath *corev1.HostPathVolumeSource `json:"hostPath,omitempty"` + HostPath *HostPathMediumSource `json:"hostPath,omitempty"` - // EmptyDir represents a temporary directory that shares a pod's lifetime. - // More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + // High is the ratio of high watermark of the tier (e.g., "0.9"). + // When cache usage exceeds this ratio, eviction will be triggered. // +optional - EmptyDir *corev1.EmptyDirVolumeSource `json:"emptyDir,omitempty"` + High string `json:"high,omitempty"` - // Ephemeral represents a volume that is handled by a cluster storage driver. - // The volume's lifecycle is tied to the pod that defines it. - // More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/ + // Low is the ratio of low watermark of the tier (e.g., "0.7"). + // Eviction will continue until cache usage falls below this ratio. + // Type specifies the type of hostPath volume. + // Defaults to empty string (no validation). + // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath // +optional - Ephemeral *corev1.EphemeralVolumeSource `json:"ephemeral,omitempty"` + Low string `json:"low,omitempty"` } func (runtime *CacheRuntime) GetStatus() *CacheRuntimeStatus { diff --git a/api/v1alpha1/openapi_generated.go b/api/v1alpha1/openapi_generated.go index e836541ec22..4237a0c92ad 100644 --- a/api/v1alpha1/openapi_generated.go +++ b/api/v1alpha1/openapi_generated.go @@ -79,6 +79,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/fluid-cloudnative/fluid/api/v1alpha1.EFCRuntime": schema_fluid_cloudnative_fluid_api_v1alpha1_EFCRuntime(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.EFCRuntimeList": schema_fluid_cloudnative_fluid_api_v1alpha1_EFCRuntimeList(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.EFCRuntimeSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_EFCRuntimeSpec(ref), + "github.com/fluid-cloudnative/fluid/api/v1alpha1.EmptyDirMediumSource": schema_fluid_cloudnative_fluid_api_v1alpha1_EmptyDirMediumSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.EncryptOption": schema_fluid_cloudnative_fluid_api_v1alpha1_EncryptOption(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.EncryptOptionSource": schema_fluid_cloudnative_fluid_api_v1alpha1_EncryptOptionSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.ExecutionCommonEntry": schema_fluid_cloudnative_fluid_api_v1alpha1_ExecutionCommonEntry(ref), @@ -88,6 +89,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/fluid-cloudnative/fluid/api/v1alpha1.ExtraResourcesComponentDependency": schema_fluid_cloudnative_fluid_api_v1alpha1_ExtraResourcesComponentDependency(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.HCFSStatus": schema_fluid_cloudnative_fluid_api_v1alpha1_HCFSStatus(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.HeadlessRuntimeComponentService": schema_fluid_cloudnative_fluid_api_v1alpha1_HeadlessRuntimeComponentService(ref), + "github.com/fluid-cloudnative/fluid/api/v1alpha1.HostPathMediumSource": schema_fluid_cloudnative_fluid_api_v1alpha1_HostPathMediumSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.InitFuseSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_InitFuseSpec(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.InitUsersSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_InitUsersSpec(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.JindoCompTemplateSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_JindoCompTemplateSpec(ref), @@ -103,7 +105,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/fluid-cloudnative/fluid/api/v1alpha1.JuiceFSRuntimeSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_JuiceFSRuntimeSpec(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.Level": schema_fluid_cloudnative_fluid_api_v1alpha1_Level(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.MasterSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_MasterSpec(ref), - "github.com/fluid-cloudnative/fluid/api/v1alpha1.MediumSource": schema_fluid_cloudnative_fluid_api_v1alpha1_MediumSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.Metadata": schema_fluid_cloudnative_fluid_api_v1alpha1_Metadata(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.MetadataSyncPolicy": schema_fluid_cloudnative_fluid_api_v1alpha1_MetadataSyncPolicy(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.Mount": schema_fluid_cloudnative_fluid_api_v1alpha1_Mount(ref), @@ -153,7 +154,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/fluid-cloudnative/fluid/api/v1alpha1.VineyardRuntime": schema_fluid_cloudnative_fluid_api_v1alpha1_VineyardRuntime(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.VineyardRuntimeList": schema_fluid_cloudnative_fluid_api_v1alpha1_VineyardRuntimeList(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.VineyardRuntimeSpec": schema_fluid_cloudnative_fluid_api_v1alpha1_VineyardRuntimeSpec(ref), - "github.com/fluid-cloudnative/fluid/api/v1alpha1.VolumeMediumSource": schema_fluid_cloudnative_fluid_api_v1alpha1_VolumeMediumSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.VolumeSource": schema_fluid_cloudnative_fluid_api_v1alpha1_VolumeSource(ref), "github.com/fluid-cloudnative/fluid/api/v1alpha1.WaitingStatus": schema_fluid_cloudnative_fluid_api_v1alpha1_WaitingStatus(ref), } @@ -3788,6 +3788,35 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_EFCRuntimeSpec(ref common.Refer } } +func schema_fluid_cloudnative_fluid_api_v1alpha1_EmptyDirMediumSource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "EmptyDirMediumSource describes emptyDir volume as a storage medium. Can be backed by node's default storage or memory (tmpfs).", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "quota": { + SchemaProps: spec.SchemaProps{ + Description: "Quota specifies the maximum storage capacity for the emptyDir volume. For Memory medium, this limit is also constrained by the sum of container memory limits. The actual limit = min(Quota, sum of container memory limits). Example: \"100Gi\" limits the emptyDir to 100GiB.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + "medium": { + SchemaProps: spec.SchemaProps{ + Description: "medium represents what type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"quota"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + func schema_fluid_cloudnative_fluid_api_v1alpha1_EncryptOption(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -4056,6 +4085,57 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_HeadlessRuntimeComponentService } } +func schema_fluid_cloudnative_fluid_api_v1alpha1_HostPathMediumSource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "HostPathMediumSource describes hostPath volumes as a storage medium. Multiple paths can be configured to distribute cache across different disks.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "paths": { + SchemaProps: spec.SchemaProps{ + Description: "Paths is a list of file paths on the host machine to be used for caching. Multiple paths allow distributing cache across different mount points or disks. Example: [\"/mnt/cache1\", \"/mnt/cache2\"]", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "quotas": { + SchemaProps: spec.SchemaProps{ + Description: "Quotas is a list of storage quotas corresponding to each path. The length of Quotas must match the length of Paths. Each quota defines the maximum cache size for the corresponding path. Example: [\"100Gi\", \"50Gi\"] allocates 100GiB to the first path and 50GiB to the second.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + "type": { + SchemaProps: spec.SchemaProps{ + Description: "Type specifies the type of hostPath volume. Defaults to empty string (no validation). More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"paths", "quotas"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + func schema_fluid_cloudnative_fluid_api_v1alpha1_InitFuseSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5499,33 +5579,6 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_MasterSpec(ref common.Reference } } -func schema_fluid_cloudnative_fluid_api_v1alpha1_MediumSource(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "MediumSource describes the storage medium type for tiered store. Only one of its members may be specified.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "processMemory": { - SchemaProps: spec.SchemaProps{ - Description: "ProcessMemory indicates that process memory should be used as the storage medium. The cache will be stored in the process's memory space.", - Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.ProcessMemoryMediumSource"), - }, - }, - "volume": { - SchemaProps: spec.SchemaProps{ - Description: "Volume indicates that a Kubernetes volume should be used as the storage medium. Supported volume types include hostPath, emptyDir, and ephemeral volumes.", - Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.VolumeMediumSource"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "github.com/fluid-cloudnative/fluid/api/v1alpha1.ProcessMemoryMediumSource", "github.com/fluid-cloudnative/fluid/api/v1alpha1.VolumeMediumSource"}, - } -} - func schema_fluid_cloudnative_fluid_api_v1alpha1_Metadata(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5967,10 +6020,21 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_ProcessMemoryMediumSource(ref c return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "ProcessMemoryMediumSource describes process memory as a storage medium. When specified, cache data will be stored in the process's memory space, and the quota will be added to the container's resource requests and limits.", + Description: "ProcessMemoryMediumSource describes process memory as a storage medium. Cache data will be stored in the process's memory space.", Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "quota": { + SchemaProps: spec.SchemaProps{ + Description: "Quota specifies the amount of memory to allocate for caching. This value will be added to the container's resource requests and limits. Example: \"4Gi\" allocates 4GiB memory for cache.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + Required: []string{"quota"}, }, }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } @@ -6851,42 +6915,25 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_RuntimeTieredStoreLevel(ref com return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "RuntimeTieredStoreLevel describes the configuration for a single tier in the tiered storage", + Description: "RuntimeTieredStoreLevel describes the configuration for a single tier in the tiered storage. Each tier can use different storage media (e.g., memory, SSD, HDD). Only one of ProcessMemory, EmptyDir, or HostPath should be specified.", Type: []string{"object"}, Properties: map[string]spec.Schema{ - "medium": { + "processMemory": { SchemaProps: spec.SchemaProps{ - Description: "Medium describes the storage medium type for this tier. Supported types include process memory and various volume types.", - Default: map[string]interface{}{}, - Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.MediumSource"), + Description: "ProcessMemory indicates that process memory should be used as one storage medium. When specified, cache data will be stored in the process's memory space, and the quota will be added to the container's resource requests and limits.", + Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.ProcessMemoryMediumSource"), }, }, - "path": { + "emptyDir": { SchemaProps: spec.SchemaProps{ - Description: "Path is a list of file paths to be used for the cache tier. Multiple paths can be specified to distribute cache across different mount points. For example: [\"/mnt/cache1\", \"/mnt/cache2\"].", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, + Description: "EmptyDir indicates that an emptyDir volume should be used as the storage medium.", + Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.EmptyDirMediumSource"), }, }, - "quota": { + "hostPath": { SchemaProps: spec.SchemaProps{ - Description: "Quota is a list of storage quotas for each path in the tier. The length of Quota should match the length of Path. Each quota corresponds to the path at the same index. For example: [\"100Gi\", \"50Gi\"] allocates 100GiB to the first path and 50GiB to the second path.", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), - }, - }, - }, + Description: "HostPath indicates that one or more hostPath volumes should be used as a storage medium.", + Ref: ref("github.com/fluid-cloudnative/fluid/api/v1alpha1.HostPathMediumSource"), }, }, "high": { @@ -6898,7 +6945,7 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_RuntimeTieredStoreLevel(ref com }, "low": { SchemaProps: spec.SchemaProps{ - Description: "Low is the ratio of low watermark of the tier (e.g., \"0.7\"). Eviction will continue until cache usage falls below this ratio.", + Description: "Low is the ratio of low watermark of the tier (e.g., \"0.7\"). Eviction will continue until cache usage falls below this ratio. Type specifies the type of hostPath volume. Defaults to empty string (no validation). More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", Type: []string{"string"}, Format: "", }, @@ -6907,7 +6954,7 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_RuntimeTieredStoreLevel(ref com }, }, Dependencies: []string{ - "github.com/fluid-cloudnative/fluid/api/v1alpha1.MediumSource", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + "github.com/fluid-cloudnative/fluid/api/v1alpha1.EmptyDirMediumSource", "github.com/fluid-cloudnative/fluid/api/v1alpha1.HostPathMediumSource", "github.com/fluid-cloudnative/fluid/api/v1alpha1.ProcessMemoryMediumSource"}, } } @@ -8449,39 +8496,6 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_VineyardRuntimeSpec(ref common. } } -func schema_fluid_cloudnative_fluid_api_v1alpha1_VolumeMediumSource(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "VolumeMediumSource describes a Kubernetes volume as a storage medium. Only one of its members may be specified.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "hostPath": { - SchemaProps: spec.SchemaProps{ - Description: "HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", - Ref: ref("k8s.io/api/core/v1.HostPathVolumeSource"), - }, - }, - "emptyDir": { - SchemaProps: spec.SchemaProps{ - Description: "EmptyDir represents a temporary directory that shares a pod's lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", - Ref: ref("k8s.io/api/core/v1.EmptyDirVolumeSource"), - }, - }, - "ephemeral": { - SchemaProps: spec.SchemaProps{ - Description: "Ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/", - Ref: ref("k8s.io/api/core/v1.EphemeralVolumeSource"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "k8s.io/api/core/v1.EmptyDirVolumeSource", "k8s.io/api/core/v1.EphemeralVolumeSource", "k8s.io/api/core/v1.HostPathVolumeSource"}, - } -} - func schema_fluid_cloudnative_fluid_api_v1alpha1_VolumeSource(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dea16e731b2..d5a812ef715 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2318,31 +2318,6 @@ func (in *MasterSpec) DeepCopy() *MasterSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MediumSource) DeepCopyInto(out *MediumSource) { - *out = *in - if in.ProcessMemory != nil { - in, out := &in.ProcessMemory, &out.ProcessMemory - *out = new(ProcessMemoryMediumSource) - **out = **in - } - if in.Volume != nil { - in, out := &in.Volume, &out.Volume - *out = new(VolumeMediumSource) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MediumSource. -func (in *MediumSource) DeepCopy() *MediumSource { - if in == nil { - return nil - } - out := new(MediumSource) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metadata) DeepCopyInto(out *Metadata) { *out = *in @@ -2556,6 +2531,54 @@ func (in *ProcessMemoryMediumSource) DeepCopy() *ProcessMemoryMediumSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EmptyDirMediumSource) DeepCopyInto(out *EmptyDirMediumSource) { + *out = *in + out.Quota = in.Quota.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyDirMediumSource. +func (in *EmptyDirMediumSource) DeepCopy() *EmptyDirMediumSource { + if in == nil { + return nil + } + out := new(EmptyDirMediumSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostPathMediumSource) DeepCopyInto(out *HostPathMediumSource) { + *out = *in + if in.Paths != nil { + in, out := &in.Paths, &out.Paths + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Quotas != nil { + in, out := &in.Quotas, &out.Quotas + *out = make([]resource.Quantity, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(v1.HostPathType) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMediumSource. +func (in *HostPathMediumSource) DeepCopy() *HostPathMediumSource { + if in == nil { + return nil + } + out := new(HostPathMediumSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Processor) DeepCopyInto(out *Processor) { *out = *in @@ -2905,18 +2928,20 @@ func (in *RuntimeTieredStore) DeepCopy() *RuntimeTieredStore { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RuntimeTieredStoreLevel) DeepCopyInto(out *RuntimeTieredStoreLevel) { *out = *in - in.Medium.DeepCopyInto(&out.Medium) - if in.Path != nil { - in, out := &in.Path, &out.Path - *out = make([]string, len(*in)) - copy(*out, *in) + if in.ProcessMemory != nil { + in, out := &in.ProcessMemory, &out.ProcessMemory + *out = new(ProcessMemoryMediumSource) + **out = **in } - if in.Quota != nil { - in, out := &in.Quota, &out.Quota - *out = make([]resource.Quantity, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + if in.EmptyDir != nil { + in, out := &in.EmptyDir, &out.EmptyDir + *out = new(EmptyDirMediumSource) + (*in).DeepCopyInto(*out) + } + if in.HostPath != nil { + in, out := &in.HostPath, &out.HostPath + *out = new(HostPathMediumSource) + (*in).DeepCopyInto(*out) } } @@ -3645,36 +3670,6 @@ func (in *VineyardRuntimeSpec) DeepCopy() *VineyardRuntimeSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VolumeMediumSource) DeepCopyInto(out *VolumeMediumSource) { - *out = *in - if in.HostPath != nil { - in, out := &in.HostPath, &out.HostPath - *out = new(v1.HostPathVolumeSource) - (*in).DeepCopyInto(*out) - } - if in.EmptyDir != nil { - in, out := &in.EmptyDir, &out.EmptyDir - *out = new(v1.EmptyDirVolumeSource) - (*in).DeepCopyInto(*out) - } - if in.Ephemeral != nil { - in, out := &in.Ephemeral, &out.Ephemeral - *out = new(v1.EphemeralVolumeSource) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMediumSource. -func (in *VolumeMediumSource) DeepCopy() *VolumeMediumSource { - if in == nil { - return nil - } - out := new(VolumeMediumSource) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSource) DeepCopyInto(out *VolumeSource) { *out = *in diff --git a/charts/fluid/fluid/crds/data.fluid.io_cacheruntimes.yaml b/charts/fluid/fluid/crds/data.fluid.io_cacheruntimes.yaml index 9f749c881b7..261cba9ae89 100644 --- a/charts/fluid/fluid/crds/data.fluid.io_cacheruntimes.yaml +++ b/charts/fluid/fluid/crds/data.fluid.io_cacheruntimes.yaml @@ -179,146 +179,56 @@ spec: levels: items: properties: + emptyDir: + properties: + medium: + type: string + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota + type: object high: type: string + hostPath: + properties: + paths: + items: + type: string + minItems: 1 + type: array + quotas: + items: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minItems: 1 + type: array + type: + type: string + required: + - paths + - quotas + type: object low: type: string - medium: + processMemory: properties: - processMemory: - type: object - volume: - properties: - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - type: object + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota type: object - path: - items: - type: string - minItems: 1 - type: array - quota: - items: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: array type: object type: array type: object @@ -1440,146 +1350,56 @@ spec: levels: items: properties: + emptyDir: + properties: + medium: + type: string + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota + type: object high: type: string + hostPath: + properties: + paths: + items: + type: string + minItems: 1 + type: array + quotas: + items: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minItems: 1 + type: array + type: + type: string + required: + - paths + - quotas + type: object low: type: string - medium: + processMemory: properties: - processMemory: - type: object - volume: - properties: - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - type: object + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota type: object - path: - items: - type: string - minItems: 1 - type: array - quota: - items: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: array type: object type: array type: object diff --git a/config/crd/bases/data.fluid.io_cacheruntimes.yaml b/config/crd/bases/data.fluid.io_cacheruntimes.yaml index 9f749c881b7..261cba9ae89 100644 --- a/config/crd/bases/data.fluid.io_cacheruntimes.yaml +++ b/config/crd/bases/data.fluid.io_cacheruntimes.yaml @@ -179,146 +179,56 @@ spec: levels: items: properties: + emptyDir: + properties: + medium: + type: string + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota + type: object high: type: string + hostPath: + properties: + paths: + items: + type: string + minItems: 1 + type: array + quotas: + items: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minItems: 1 + type: array + type: + type: string + required: + - paths + - quotas + type: object low: type: string - medium: + processMemory: properties: - processMemory: - type: object - volume: - properties: - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - type: object + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota type: object - path: - items: - type: string - minItems: 1 - type: array - quota: - items: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: array type: object type: array type: object @@ -1440,146 +1350,56 @@ spec: levels: items: properties: + emptyDir: + properties: + medium: + type: string + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota + type: object high: type: string + hostPath: + properties: + paths: + items: + type: string + minItems: 1 + type: array + quotas: + items: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + minItems: 1 + type: array + type: + type: string + required: + - paths + - quotas + type: object low: type: string - medium: + processMemory: properties: - processMemory: - type: object - volume: - properties: - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - metadata: - type: object - spec: - properties: - accessModes: - items: - type: string - type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - type: string - volumeAttributesClassName: - type: string - volumeMode: - type: string - volumeName: - type: string - type: object - required: - - spec - type: object - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - type: object + quota: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - quota type: object - path: - items: - type: string - minItems: 1 - type: array - quota: - items: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: array type: object type: array type: object diff --git a/docs/zh/userguide/cache_runtime_tieredstore.md b/docs/zh/userguide/cache_runtime_tieredstore.md new file mode 100644 index 00000000000..06a987784af --- /dev/null +++ b/docs/zh/userguide/cache_runtime_tieredstore.md @@ -0,0 +1,280 @@ +# CacheRuntime TieredStore 配置示例 + +本文档展示了如何为 CacheRuntime 配置分层存储（TieredStore）。 + +## 概述 + +CacheRuntime 支持两种类型的存储介质： +1. **ProcessMemory**：使用进程内存作为缓存存储 +2. **Volume**：使用 Kubernetes Volume 作为缓存存储（支持 hostPath、emptyDir、ephemeral） + +## 示例 1：使用 ProcessMemory + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - medium: + processMemory: {} + path: + - /dev/shm + quota: + - 8Gi + high: "0.95" + low: "0.7" +``` + +**说明：** +- `processMemory: {}` 表示使用进程内存 +- `path` 指定缓存路径（对于内存，通常是 /dev/shm） +- `quota` 设置内存配额，会自动添加到容器的 resource requests/limits +- `high` 和 `low` 是水位线配置，用于控制缓存驱逐 + +## 示例 2：使用 EmptyDir Volume + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - medium: + volume: + emptyDir: + medium: "" # 使用磁盘空间，如果要使用内存设置为 "Memory" + path: + - /mnt/cache + quota: + - 100Gi + high: "0.95" + low: "0.7" +``` + +**说明：** +- `emptyDir` 会创建一个临时目录，生命周期与 Pod 相同 +- `quota` 会设置为 EmptyDir 的 sizeLimit +- 缓存数据会在 Pod 重启后丢失 + +## 示例 3：使用 HostPath Volume + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - medium: + volume: + hostPath: + path: /data/cache + type: DirectoryOrCreate + path: + - /mnt/cache + high: "0.95" + low: "0.7" +``` + +**说明：** +- `hostPath` 使用节点上的持久化目录 +- 缓存数据在 Pod 重启后仍然保留 +- 需要确保节点上有相应的目录权限 + +## 示例 4：多层级存储 + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + # 第一层：高速内存缓存 + - medium: + processMemory: {} + path: + - /dev/shm + quota: + - 4Gi + high: "0.95" + low: "0.7" + + # 第二层：大容量磁盘缓存 + - medium: + volume: + emptyDir: {} + path: + - /mnt/ssd + quota: + - 200Gi + high: "0.90" + low: "0.6" +``` + +**说明：** +- 可以配置多个层级，按优先级从高到低排列 +- 每层可以有不同的介质类型、配额和水位线 +- 系统会优先使用高层级的存储 + +## 示例 5：多路径配置 + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - medium: + processMemory: {} + path: + - /dev/shm/cache1 + - /dev/shm/cache2 + quota: + - 2Gi + - 3Gi + high: "0.95" + low: "0.7" +``` + +**说明：** +- 可以为同一层级指定多个路径 +- 每个路径可以有独立的配额 +- 总内存配额 = 2Gi + 3Gi = 5Gi + +## 示例 6：使用 Ephemeral Volume + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - medium: + volume: + ephemeral: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "fast-ssd" + path: + - /mnt/cache + quota: + - 100Gi + high: "0.95" + low: "0.7" +``` + +**说明：** +- `ephemeral` 会创建临时的 PVC，生命周期与 Pod 相同 +- `quota` 会设置为 PVC 的 storage request +- 可以使用不同的 storageClassName 来获得不同的存储性能 +- 缓存数据会在 Pod 删除后丢失，但比 EmptyDir 更灵活 + +## 实现细节 + +### ProcessMemory 处理 + +当使用 `processMemory` 时： +1. 系统会计算所有路径的配额总和 +2. 将总配额添加到容器的 `resources.requests.memory` 和 `resources.limits.memory` +3. 如果容器已有其他资源请求，会累加而不是覆盖 + +### Volume 处理 + +当使用 `volume` 时： +1. 为每个路径创建一个独立的 Volume +2. Volume 名称格式：`tieredstore-level{N}-path{M}` +3. 根据 Volume 类型（hostPath/emptyDir/ephemeral）创建对应的 VolumeSource +4. 为容器添加对应的 VolumeMount +5. 配额处理方式因类型而异： + - **EmptyDir**：quota 设置为 `sizeLimit` + - **Ephemeral**：quota 设置为 `volumeClaimTemplate.spec.resources.requests.storage` + - **HostPath**：quota **仅供参考**，不会强制限制（需要依赖外部机制如 XFS quota） + +### 多路径配置的重要说明 + +**不同 Volume 类型对多路径的支持程度：** + +| Volume 类型 | 多路径合理性 | Quota 是否生效 | 物理隔离 | 建议 | +|------------|------------|--------------|---------|------| +| **EmptyDir** | ✅ 合理 | ✅ 是（通过 sizeLimit） | ✅ 独立 tmpfs/disk | 推荐使用 | +| **Ephemeral** | ✅ 合理 | ✅ 是（通过 PVC storage request） | ✅ 独立 PVC | 推荐使用 | +| **HostPath** | ❌ 不推荐 | ❌ 否（共享节点存储） | ❌ 无隔离 | 建议使用单路径 | +| **ProcessMemory** | ⚠️ 有限 | ⚠️ 部分（累加到容器 limit） | ❌ 无隔离 | 谨慎使用多路径 | + +**详细说明：** + +1. **EmptyDir + 多路径**： + - ✅ 每个路径创建独立的 EmptyDir volume + - ✅ 每个 volume 有独立的 `sizeLimit` + - ✅ 真正的物理隔离和配额控制 + - 示例：可以使用多个磁盘分区或 tmpfs + +2. **Ephemeral + 多路径**： + - ✅ 每个路径创建独立的 PVC + - ✅ 每个 PVC 可以有独立的 storage request（通过 quota 设置） + - ⚠️ **所有 PVC 共享相同的 volumeClaimTemplate 配置**（包括 storageClassName、accessModes 等） + - ⚠️ **无法在同一个 Level 内混合使用不同的 storage class** + - 💡 多路径的主要用途：**扩展同一类型存储的总容量** + - 示例：使用多个 SSD PVC，每个 100Gi，总容量 200Gi + - 📌 如需使用不同类型的存储（SSD + HDD），应配置**不同的 Level** + +3. **HostPath + 多路径**： + - ❌ 多个路径可能指向同一节点的存储空间 + - ❌ quota 无法直接生效，只是配置元数据 + - ❌ 没有真正的配额限制和物理隔离 + - ⚠️ 如果需要配额控制，请使用操作系统级别的 quota 机制 + - **建议**：HostPath 场景下只配置单个路径 + +4. **ProcessMemory + 多路径**： + - ⚠️ 所有路径的 quota 会累加到容器的 memory requests/limits + - ⚠️ 路径之间没有物理隔离（都在 /dev/shm 下） + - ⚠️ 应用程序可以随意使用总内存，不受单个路径 quota 限制 + - **建议**：除非有特殊的逻辑分区需求，否则使用单路径即可 + +## 注意事项 + +1. **配额单位**：支持 Kubernetes 标准单位（Ki, Mi, Gi, Ti 等） +2. **水位线**：`high` 和 `low` 是 0-1 之间的小数，表示使用率的百分比 +3. **路径要求**：`path` 数组和 `quota` 数组长度必须一致 +4. **介质选择**：每个层级只能选择一种介质类型（ProcessMemory 或 Volume） +5. **Volume 限制**：每个层级的 Volume 只能选择一种类型（hostPath、emptyDir 或 ephemeral） +6. **多路径建议**： + - EmptyDir 和 Ephemeral 类型推荐使用多路径，可以实现真正的配额控制和物理隔离 + - HostPath 类型建议使用单路径，因为 quota 无法生效 + - ProcessMemory 类型谨慎使用多路径，只有逻辑分区意义，无物理隔离 +7. **配额生效机制**： + - EmptyDir：通过 `sizeLimit` 强制限制 + - Ephemeral：通过 PVC 的 storage request 限制 + - HostPath：需要依赖外部机制（如 XFS project quota、Cgroup 等） + - ProcessMemory：通过容器 memory limit 限制总量，但路径间无隔离 diff --git a/pkg/common/cacheruntime.go b/pkg/common/cacheruntime.go index f0d8b6e92e5..56b02e69494 100644 --- a/pkg/common/cacheruntime.go +++ b/pkg/common/cacheruntime.go @@ -110,6 +110,10 @@ type CacheRuntimeComponentConfig struct { Replicas int32 `json:"replicas,omitempty"` Service CacheRuntimeComponentServiceConfig `json:"service,omitempty"` + + // TieredStoreLevels contains the tiered storage configuration for worker + // This field is primarily used by Worker component to configure cache storage tiers + TieredStoreLevels []TieredStoreLevelConfig `json:"tieredStoreLevels,omitempty"` } type CacheRuntimeComponentServiceConfig struct { @@ -125,3 +129,31 @@ func GetCacheComponentName(runtimeName string, componentType ComponentType) stri func GetCacheRuntimeConfigConfigMapName(name string) string { return fmt.Sprintf("fluid-runtime-config-%s", name) } + +// TieredStoreLevelConfig defines the configuration for a single tier in the tiered storage. +// This config will be mounted into the worker container via ConfigMap. +type TieredStoreLevelConfig struct { + // MountPaths contains the mount paths inside the container for this tier + // For processMemory: empty array (no mount path) + // For emptyDir: single element array with the mount path + // For hostPath: array of mount paths corresponding to each host path + MountPaths []string `json:"mountPaths,omitempty"` + + // MediumType indicates the storage medium type: "MEM", "SSD", "HDD", etc. + // This represents the performance characteristics of the storage medium + MediumType MediumType `json:"mediumType,omitempty"` + + // Quotas is the storage capacity for this tier (e.g., ["100Gi"]) + // The length of Quotas matches the length of MountPaths + // For processMemory/emptyDir: single element array with the quota + // For hostPath: array of quotas corresponding to each mount path + Quotas []string `json:"quotas,omitempty"` + + // High is the high watermark ratio (e.g., "0.9") + // When cache usage exceeds this ratio, eviction will be triggered + High string `json:"high,omitempty"` + + // Low is the low watermark ratio (e.g., "0.7") + // Eviction will continue until cache usage falls below this ratio + Low string `json:"low,omitempty"` +} diff --git a/pkg/ddc/cache/engine/cm.go b/pkg/ddc/cache/engine/cm.go index 21d74391ff1..8218f09ff5a 100644 --- a/pkg/ddc/cache/engine/cm.go +++ b/pkg/ddc/cache/engine/cm.go @@ -166,6 +166,8 @@ func (e *CacheEngine) generateRuntimeConfigData(ctx context.Context, runtime *da Name: GetComponentServiceName(e.name, common.ComponentTypeWorker), } } + // Extract tiered store configuration for worker + config.Worker.TieredStoreLevels = e.extractTieredStoreLevels(&runtime.Spec.Worker.TieredStore) } if !runtime.Spec.Client.Disabled { config.Client = &common.CacheRuntimeComponentConfig{ @@ -180,6 +182,8 @@ func (e *CacheEngine) generateRuntimeConfigData(ctx context.Context, runtime *da Name: GetComponentServiceName(e.name, common.ComponentTypeClient), } } + // Extract tiered store configuration for client + config.Client.TieredStoreLevels = e.extractTieredStoreLevels(&runtime.Spec.Client.TieredStore) } b, _ := json.Marshal(config) @@ -189,3 +193,54 @@ func (e *CacheEngine) generateRuntimeConfigData(ctx context.Context, runtime *da } return data, nil } + +// extractTieredStoreLevels extracts tiered store configuration from RuntimeTieredStore +// and converts it to TieredStoreLevelConfig for the ConfigMap +func (e *CacheEngine) extractTieredStoreLevels(tieredStore *datav1alpha1.RuntimeTieredStore) []common.TieredStoreLevelConfig { + if tieredStore == nil || len(tieredStore.Levels) == 0 { + return nil + } + + var levels []common.TieredStoreLevelConfig + + for levelIndex, level := range tieredStore.Levels { + // Determine medium type and extract corresponding configuration + levelConfig := common.TieredStoreLevelConfig{ + High: level.High, + Low: level.Low, + } + // some cache system like curvine need set the medium type + // default HDD, currently only use MEM and HDD. + levelConfig.MediumType = common.HDD + if level.ProcessMemory != nil { + levelConfig.MediumType = common.Memory + levelConfig.MountPaths = []string{GetMemoryTieredStoreMountPath(levelIndex)} + levelConfig.Quotas = []string{level.ProcessMemory.Quota.String()} + } else if level.HostPath != nil { + // Generate mount paths for all host paths + var mountPaths []string + for pathIndex := range level.HostPath.Paths { + mountPaths = append(mountPaths, GetHostPathTieredStoreMountPath(levelIndex, pathIndex)) + } + levelConfig.MountPaths = mountPaths + // Convert quotas to string array + var quotas []string + for _, quota := range level.HostPath.Quotas { + quotas = append(quotas, quota.String()) + } + levelConfig.Quotas = quotas + + } else if level.EmptyDir != nil { + // MEM medium + if level.EmptyDir.Medium == corev1.StorageMediumMemory { + levelConfig.MediumType = common.Memory + } + levelConfig.MountPaths = []string{GetEmptyDirTieredStoreMountPath(levelIndex)} + levelConfig.Quotas = []string{level.EmptyDir.Quota.String()} + } + + levels = append(levels, levelConfig) + } + + return levels +} diff --git a/pkg/ddc/cache/engine/transform_client.go b/pkg/ddc/cache/engine/transform_client.go index 76148380c06..208e1aad592 100644 --- a/pkg/ddc/cache/engine/transform_client.go +++ b/pkg/ddc/cache/engine/transform_client.go @@ -47,6 +47,12 @@ func (e *CacheEngine) transformClient(dataset *datav1alpha1.Dataset, runtime *da // transform container related config, currently only modify the first container e.transformComponentPodTemplate(runtimeClient.RuntimeComponentCommonSpec, dataset, value.Client) + // transform tiered store configuration into pod resource request or volumes . + err = e.TransformRuntimeTieredStore(&runtimeClient.TieredStore, &value.Client.PodTemplateSpec.Spec) + if err != nil { + return err + } + // transform all volume-related configurations // Client default does NOT mount secrets (defaultMountSecrets=false) err = e.transformVolumes(runtime.Spec.Volumes, runtime.Spec.Client.VolumeMounts, dataset, componentDefinition, commonConfig, false, &value.Client.PodTemplateSpec.Spec) diff --git a/pkg/ddc/cache/engine/transform_client_test.go b/pkg/ddc/cache/engine/transform_client_test.go index ecdb3a4a4e7..1fed86f22e5 100644 --- a/pkg/ddc/cache/engine/transform_client_test.go +++ b/pkg/ddc/cache/engine/transform_client_test.go @@ -1,17 +1,17 @@ /* -Copyright 2026 The Fluid Authors. + Copyright 2026 The Fluid Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ package engine diff --git a/pkg/ddc/cache/engine/transform_tiered_store.go b/pkg/ddc/cache/engine/transform_tiered_store.go new file mode 100644 index 00000000000..9069af25f1b --- /dev/null +++ b/pkg/ddc/cache/engine/transform_tiered_store.go @@ -0,0 +1,215 @@ +/* + Copyright 2026 The Fluid Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package engine + +import ( + "fmt" + + datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" + "github.com/fluid-cloudnative/fluid/pkg/utils" + corev1 "k8s.io/api/core/v1" +) + +// TransformRuntimeTieredStore transforms the tiered store configuration to worker pod spec +func (e *CacheEngine) TransformRuntimeTieredStore(tieredStore *datav1alpha1.RuntimeTieredStore, podSpec *corev1.PodSpec) error { + if len(tieredStore.Levels) == 0 { + return nil + } + + if len(podSpec.Containers) == 0 { + return fmt.Errorf("no containers found in worker pod spec") + } + + container := &podSpec.Containers[0] + + // Process each tier level + for i, level := range tieredStore.Levels { + // order: memory, host path, empty. only one can be specified per level + mediaCount := 0 + + // Process memory: add resource requests and limits + if level.ProcessMemory != nil { + mediaCount++ + err := e.handleProcessMemory(podSpec, container, level.ProcessMemory, i) + if err != nil { + return err + } + } + + // Volume-based storage: create volumes and volume mounts + if level.HostPath != nil { + mediaCount++ + err := e.handleHostPath(podSpec, container, level.HostPath, i) + if err != nil { + return err + } + } + + // EmptyDir: add volume and volume mount + if level.EmptyDir != nil { + mediaCount++ + err := e.handleEmptyDir(podSpec, container, level.EmptyDir, i) + if err != nil { + return err + } + } + + if mediaCount > 1 { + return fmt.Errorf("only one storage medium can be specified per level at index %d, but found %d", i, mediaCount) + } + } + + return nil +} + +// handleProcessMemory adds memory resources to container for process memory medium +func (e *CacheEngine) handleProcessMemory(podSpec *corev1.PodSpec, container *corev1.Container, memoryMediumSource *datav1alpha1.ProcessMemoryMediumSource, levelIndex int) error { + if memoryMediumSource.Quota.IsZero() { + return fmt.Errorf("process memory quota cannot be zero at level index %d", levelIndex) + } + + // Calculate total memory quota across all paths + totalQuota := memoryMediumSource.Quota + + // add totalQuota to memory resources only when memory is restricted. + if container.Resources.Requests != nil { + if currentRequest, exists := container.Resources.Requests[corev1.ResourceMemory]; exists && !currentRequest.IsZero() { + currentRequest.Add(totalQuota) + container.Resources.Requests[corev1.ResourceMemory] = currentRequest + } + } + if container.Resources.Limits != nil { + if currentLimit, exists := container.Resources.Limits[corev1.ResourceMemory]; exists && !currentLimit.IsZero() { + currentLimit.Add(totalQuota) + container.Resources.Limits[corev1.ResourceMemory] = currentLimit + } + } + + // add an memory emptyDir for /dev/shm in the container + volumeName := fmt.Sprintf("tiered-store-level-%d-memory", levelIndex) + mountPath := GetEmptyDirTieredStoreMountPath(levelIndex) + volume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: &totalQuota, + }, + }, + } + + // Add volume to pod spec + podSpec.Volumes = utils.AppendOrOverrideVolume(podSpec.Volumes, volume) + + // Add volume mount to container + volumeMount := corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + } + container.VolumeMounts = utils.AppendOrOverrideVolumeMounts(container.VolumeMounts, volumeMount) + + return nil +} + +// handleHostPath adds volume and volume mount for volume-based medium +func (e *CacheEngine) handleHostPath(podSpec *corev1.PodSpec, container *corev1.Container, + hostPathMediumSource *datav1alpha1.HostPathMediumSource, levelIndex int) error { + + if len(hostPathMediumSource.Paths) != len(hostPathMediumSource.Quotas) { + return fmt.Errorf("number of paths and quotas must be equal at level index %d", levelIndex) + } + + // Process each path and corresponding quota + for i, hostPath := range hostPathMediumSource.Paths { + volumeName := fmt.Sprintf("tiered-store-level-%d-index-%d", levelIndex, i) + mountPath := GetHostPathTieredStoreMountPath(levelIndex, i) + + volume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath, + Type: hostPathMediumSource.Type, + }, + }, + } + + // Add volume to pod spec + podSpec.Volumes = utils.AppendOrOverrideVolume(podSpec.Volumes, volume) + + // Add volume mount to container + volumeMount := corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + } + container.VolumeMounts = utils.AppendOrOverrideVolumeMounts(container.VolumeMounts, volumeMount) + } + + return nil +} + +func (e *CacheEngine) handleEmptyDir(podSpec *corev1.PodSpec, container *corev1.Container, + emptyDirMediumSource *datav1alpha1.EmptyDirMediumSource, levelIndex int) error { + + if emptyDirMediumSource.Quota.IsZero() { + return fmt.Errorf("emptyDir quota cannot be zero for empty dir medium source at level index %d", levelIndex) + } + + volumeName := fmt.Sprintf("tiered-store-level-%d-index-%d", levelIndex, 0) + mountPath := GetEmptyDirTieredStoreMountPath(levelIndex) + + volume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: emptyDirMediumSource.Medium, + SizeLimit: &emptyDirMediumSource.Quota, + }, + }, + } + + // Add volume to pod spec + podSpec.Volumes = utils.AppendOrOverrideVolume(podSpec.Volumes, volume) + + // Add volume mount to container + volumeMount := corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + } + container.VolumeMounts = utils.AppendOrOverrideVolumeMounts(container.VolumeMounts, volumeMount) + + // For Memory-backed EmptyDir (tmpfs), add quota to container memory resources + // This ensures proper resource accounting and prevents excessive memory usage + if emptyDirMediumSource.Medium == corev1.StorageMediumMemory { + // Only add to resources if the container already has memory constraints + // If no memory resources are set, the container is unconstrained and we don't need to add + if container.Resources.Requests != nil { + if currentRequest, exists := container.Resources.Requests[corev1.ResourceMemory]; exists && !currentRequest.IsZero() { + currentRequest.Add(emptyDirMediumSource.Quota) + container.Resources.Requests[corev1.ResourceMemory] = currentRequest + } + } + if container.Resources.Limits != nil { + if currentLimit, exists := container.Resources.Limits[corev1.ResourceMemory]; exists && !currentLimit.IsZero() { + currentLimit.Add(emptyDirMediumSource.Quota) + container.Resources.Limits[corev1.ResourceMemory] = currentLimit + } + } + } + + return nil +} diff --git a/pkg/ddc/cache/engine/transform_tiered_store_test.go b/pkg/ddc/cache/engine/transform_tiered_store_test.go new file mode 100644 index 00000000000..d7f27eb1df2 --- /dev/null +++ b/pkg/ddc/cache/engine/transform_tiered_store_test.go @@ -0,0 +1,406 @@ +/* +Copyright 2026 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +var _ = Describe("CacheEngine TransformRuntimeTieredStore Tests", Label("pkg.ddc.cache.engine.transform_tiered_store_test.go"), func() { + var ( + engine *CacheEngine + podSpec *corev1.PodSpec + ) + + BeforeEach(func() { + engine = &CacheEngine{} + + // Initialize a basic pod spec with one container + podSpec = &corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker", + Image: "test-image:latest", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, + Limits: corev1.ResourceList{}, + }, + }, + }, + } + }) + + Describe("TransformRuntimeTieredStore", func() { + Context("when tiered store has no levels", func() { + It("should return nil without error", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{}, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when pod spec has no containers", func() { + It("should return error", func() { + emptyPodSpec := &corev1.PodSpec{ + Containers: []corev1.Container{}, + } + + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("1Gi"), + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, emptyPodSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no containers found")) + }) + }) + + Context("when using ProcessMemory medium", func() { + It("should add memory resources to container", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("4Gi"), + }, + High: "0.9", + Low: "0.7", + }, + }, + } + + // Set initial memory resources + podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("2Gi") + podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify memory request is increased + memRequest := podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] + expectedRequest := resource.MustParse("6Gi") // 2Gi + 4Gi + Expect(memRequest.Cmp(expectedRequest)).To(Equal(0)) + + // Verify memory limit is increased + memLimit := podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] + expectedLimit := resource.MustParse("8Gi") // 4Gi + 4Gi + Expect(memLimit.Cmp(expectedLimit)).To(Equal(0)) + }) + + It("should return error when quota is zero", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("0"), + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("quota cannot be zero")) + }) + + It("should not modify resources when container has no memory constraints", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("4Gi"), + }, + }, + }, + } + + // Container has no memory resources set + podSpec.Containers[0].Resources.Requests = nil + podSpec.Containers[0].Resources.Limits = nil + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Resources should remain nil + Expect(podSpec.Containers[0].Resources.Requests).To(BeNil()) + Expect(podSpec.Containers[0].Resources.Limits).To(BeNil()) + }) + }) + + Context("when using HostPath medium", func() { + It("should create volumes and volume mounts for single path", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + HostPath: &datav1alpha1.HostPathMediumSource{ + Paths: []string{"/mnt/cache1"}, + Quotas: []resource.Quantity{resource.MustParse("100Gi")}, + Type: nil, + }, + High: "0.9", + Low: "0.7", + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify volume is created + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("tiered-store-level-0-index-0")) + Expect(podSpec.Volumes[0].HostPath).NotTo(BeNil()) + Expect(podSpec.Volumes[0].HostPath.Path).To(Equal("/mnt/cache1")) + + // Verify volume mount is created + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("tiered-store-level-0-index-0")) + Expect(podSpec.Containers[0].VolumeMounts[0].MountPath).To(ContainSubstring("tiered-store")) + }) + + It("should create multiple volumes and mounts for multiple paths", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + HostPath: &datav1alpha1.HostPathMediumSource{ + Paths: []string{"/mnt/cache1", "/mnt/cache2", "/mnt/cache3"}, + Quotas: []resource.Quantity{ + resource.MustParse("100Gi"), + resource.MustParse("200Gi"), + resource.MustParse("300Gi"), + }, + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify 3 volumes are created + Expect(podSpec.Volumes).To(HaveLen(3)) + for i := 0; i < 3; i++ { + Expect(podSpec.Volumes[i].Name).To(Equal("tiered-store-level-0-index-" + string(rune('0'+i)))) + Expect(podSpec.Volumes[i].HostPath.Path).To(Equal("/mnt/cache" + string(rune('1'+i)))) + } + + // Verify 3 volume mounts are created + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(3)) + }) + + It("should return error when paths and quotas count mismatch", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + HostPath: &datav1alpha1.HostPathMediumSource{ + Paths: []string{"/mnt/cache1", "/mnt/cache2"}, + Quotas: []resource.Quantity{resource.MustParse("100Gi")}, + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("number of paths and quotas must be equal")) + }) + }) + + Context("when using EmptyDir medium", func() { + It("should create volume and mount for disk-based EmptyDir", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("50Gi"), + Medium: corev1.StorageMediumDefault, + }, + High: "0.85", + Low: "0.65", + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify volume is created + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("tiered-store-level-0-index-0")) + Expect(podSpec.Volumes[0].EmptyDir).NotTo(BeNil()) + Expect(podSpec.Volumes[0].EmptyDir.Medium).To(Equal(corev1.StorageMediumDefault)) + + // Verify SizeLimit is set + Expect(podSpec.Volumes[0].EmptyDir.SizeLimit).NotTo(BeNil()) + Expect(podSpec.Volumes[0].EmptyDir.SizeLimit.String()).To(Equal("50Gi")) + + // Verify volume mount is created + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + }) + + It("should add memory resources for Memory-backed EmptyDir", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("2Gi"), + Medium: corev1.StorageMediumMemory, + }, + }, + }, + } + + // Set initial memory resources + podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Gi") + podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] = resource.MustParse("2Gi") + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify memory request is increased + memRequest := podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] + expectedRequest := resource.MustParse("3Gi") // 1Gi + 2Gi + Expect(memRequest.Cmp(expectedRequest)).To(Equal(0)) + + // Verify memory limit is increased + memLimit := podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] + expectedLimit := resource.MustParse("4Gi") // 2Gi + 2Gi + Expect(memLimit.Cmp(expectedLimit)).To(Equal(0)) + }) + + It("should return error when quota is zero", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("0"), + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("quota cannot be zero")) + }) + }) + + Context("when using multiple tier levels", func() { + It("should process all levels correctly", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + // Level 0: ProcessMemory + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("4Gi"), + }, + }, + // Level 1: HostPath + { + HostPath: &datav1alpha1.HostPathMediumSource{ + Paths: []string{"/mnt/ssd1"}, + Quotas: []resource.Quantity{resource.MustParse("100Gi")}, + }, + }, + // Level 2: EmptyDir + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("50Gi"), + }, + }, + }, + } + + // Set initial memory resources + podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("2Gi") + podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Verify 3 volumes are created (ProcessMemory's EmptyDir + HostPath + EmptyDir) + Expect(podSpec.Volumes).To(HaveLen(3)) + + // Verify 3 volume mounts are created + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(3)) + + // Verify memory resources include ProcessMemory quota + memRequest := podSpec.Containers[0].Resources.Requests[corev1.ResourceMemory] + expectedRequest := resource.MustParse("6Gi") // 2Gi + 4Gi + Expect(memRequest.Cmp(expectedRequest)).To(Equal(0)) + }) + + It("should return error when multiple media types in same level", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + // Both ProcessMemory and EmptyDir specified - invalid + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("4Gi"), + }, + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("50Gi"), + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("only one storage medium can be specified")) + }) + }) + + Context("when handling high/low watermarks", func() { + It("should preserve watermark configuration in level", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + EmptyDir: &datav1alpha1.EmptyDirMediumSource{ + Quota: resource.MustParse("100Gi"), + }, + High: "0.9", + Low: "0.7", + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).NotTo(HaveOccurred()) + + // Watermarks are stored in the tieredStore config, not in pod spec + // This test verifies that processing doesn't fail with watermarks present + Expect(tieredStore.Levels[0].High).To(Equal("0.9")) + Expect(tieredStore.Levels[0].Low).To(Equal("0.7")) + }) + }) + }) +}) diff --git a/pkg/ddc/cache/engine/transform_worker.go b/pkg/ddc/cache/engine/transform_worker.go index 916072497a5..6011f8e0cb7 100644 --- a/pkg/ddc/cache/engine/transform_worker.go +++ b/pkg/ddc/cache/engine/transform_worker.go @@ -42,11 +42,15 @@ func (e *CacheEngine) transformWorker(dataset *datav1alpha1.Dataset, runtime *da return err } - // TODO: TieredStore handling - // transform container related config, currently only modify the first container e.transformComponentPodTemplate(runtimeWorker.RuntimeComponentCommonSpec, dataset, value.Worker) + // transform tiered store configuration into pod resource request or volumes . + err = e.TransformRuntimeTieredStore(&runtimeWorker.TieredStore, &value.Worker.PodTemplateSpec.Spec) + if err != nil { + return err + } + // make sure affinity not nil if value.Worker.PodTemplateSpec.Spec.Affinity == nil { value.Worker.PodTemplateSpec.Spec.Affinity = &corev1.Affinity{} diff --git a/pkg/ddc/cache/engine/util.go b/pkg/ddc/cache/engine/util.go index 675fd550e4e..a6650e97168 100644 --- a/pkg/ddc/cache/engine/util.go +++ b/pkg/ddc/cache/engine/util.go @@ -112,3 +112,20 @@ func getSecretMountPath(secretName string) string { func getSecretFilePath(secretName, secretKey string) string { return fmt.Sprintf("%s/%s", getSecretMountPath(secretName), secretKey) } + +func GetMemoryTieredStoreMountPath(_ int) string { + return "/dev/shm" +} + +func GetHostPathTieredStoreMountPath(levelIndex int, pathIndex int) string { + return getTieredStoreMountPath(levelIndex, pathIndex, "hostpath") +} + +func GetEmptyDirTieredStoreMountPath(levelIndex int) string { + return getTieredStoreMountPath(levelIndex, 0, "emptydir") +} + +// getTieredStoreMountPath generates the volume name for a tiered store medium +func getTieredStoreMountPath(levelIndex int, pathIndex int, mediumType string) string { + return fmt.Sprintf("/etc/fluid/mount/tiered-store/level-%d-index-%d-%s", levelIndex, pathIndex, mediumType) +} diff --git a/test/gha-e2e/curvine/cacheruntime.yaml b/test/gha-e2e/curvine/cacheruntime.yaml index deb4ea7474c..56b82a856c5 100644 --- a/test/gha-e2e/curvine/cacheruntime.yaml +++ b/test/gha-e2e/curvine/cacheruntime.yaml @@ -17,13 +17,8 @@ spec: levels: #worker缓存配置 - low: "0.5" high: "0.8" - path: - - "testing/data" - quota: - - 1Gi - medium: - volume: - emptyDir: {} + emptyDir: + quota: 1Gi client: options: key1: value1 diff --git a/test/gha-e2e/curvine/cacheruntimeclass.yaml b/test/gha-e2e/curvine/cacheruntimeclass.yaml index 17d75f76f3e..5f69ab17447 100644 --- a/test/gha-e2e/curvine/cacheruntimeclass.yaml +++ b/test/gha-e2e/curvine/cacheruntimeclass.yaml @@ -27,7 +27,12 @@ extraResources: [worker] dir_reserved = "0" data_dir = [ - "[DISK]testing/data" + {{- range $index, $level := (ds "config").worker.tieredStoreLevels }} + {{- range $pathIndex, $mountPath := $level.mountPaths }} + {{- if or $index $pathIndex }},{{ end }} + "[{{$level.mediumType}}:{{ index $level.quotas $pathIndex }}]{{ $mountPath }}" + {{- end }} + {{- end }} ] dataOperationSpecs: - name: DataLoad From 1e98dd87c534c3e56b640f5400758512216d4999 Mon Sep 17 00:00:00 2001 From: xliuqq Date: Thu, 4 Jun 2026 20:10:43 +0800 Subject: [PATCH 2/7] fix yaml Signed-off-by: xliuqq --- test/gha-e2e/curvine/cacheruntimeclass.yaml | 4 ++-- test/gha-e2e/curvine/dataset.yaml | 8 -------- test/gha-e2e/curvine/minio.yaml | 8 ++++++++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/gha-e2e/curvine/cacheruntimeclass.yaml b/test/gha-e2e/curvine/cacheruntimeclass.yaml index 5f69ab17447..9d35f1e5786 100644 --- a/test/gha-e2e/curvine/cacheruntimeclass.yaml +++ b/test/gha-e2e/curvine/cacheruntimeclass.yaml @@ -23,14 +23,14 @@ extraResources: ] journal_dir = "testing/journal" - # Worker configuration + # Worker configuration, translate Gi to GB [worker] dir_reserved = "0" data_dir = [ {{- range $index, $level := (ds "config").worker.tieredStoreLevels }} {{- range $pathIndex, $mountPath := $level.mountPaths }} {{- if or $index $pathIndex }},{{ end }} - "[{{$level.mediumType}}:{{ index $level.quotas $pathIndex }}]{{ $mountPath }}" + "[{{$level.mediumType}}:{{ index $level.quotas $pathIndex | strings.ReplaceAll "i" "B"}}]{{ $mountPath }}" {{- end }} {{- end }} ] diff --git a/test/gha-e2e/curvine/dataset.yaml b/test/gha-e2e/curvine/dataset.yaml index bff5b482e65..2fc2e8235d0 100644 --- a/test/gha-e2e/curvine/dataset.yaml +++ b/test/gha-e2e/curvine/dataset.yaml @@ -1,11 +1,3 @@ -apiVersion: v1 -kind: Secret -metadata: - name: curvine-secret -stringData: - access-key: minioadmin - secret-key: minioadmin ---- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: diff --git a/test/gha-e2e/curvine/minio.yaml b/test/gha-e2e/curvine/minio.yaml index 608ab2d6008..c2ace511a74 100644 --- a/test/gha-e2e/curvine/minio.yaml +++ b/test/gha-e2e/curvine/minio.yaml @@ -1,4 +1,12 @@ apiVersion: v1 +kind: Secret +metadata: + name: curvine-secret +stringData: + access-key: minioadmin + secret-key: minioadmin +--- +apiVersion: v1 kind: Service metadata: name: minio From 3f9a6478c89e479492e465a17387a0a79bcad835 Mon Sep 17 00:00:00 2001 From: xliuqq Date: Thu, 11 Jun 2026 21:07:48 +0800 Subject: [PATCH 3/7] fix reviews Signed-off-by: xliuqq --- api/v1alpha1/cacheruntime_types.go | 3 --- api/v1alpha1/openapi_generated.go | 2 +- pkg/common/cacheruntime.go | 2 +- pkg/ddc/cache/engine/transform_tiered_store.go | 2 +- pkg/ddc/cache/engine/util.go | 2 +- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/api/v1alpha1/cacheruntime_types.go b/api/v1alpha1/cacheruntime_types.go index 524a497e1fd..51850a8a0f0 100644 --- a/api/v1alpha1/cacheruntime_types.go +++ b/api/v1alpha1/cacheruntime_types.go @@ -294,9 +294,6 @@ type RuntimeTieredStoreLevel struct { // Low is the ratio of low watermark of the tier (e.g., "0.7"). // Eviction will continue until cache usage falls below this ratio. - // Type specifies the type of hostPath volume. - // Defaults to empty string (no validation). - // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath // +optional Low string `json:"low,omitempty"` } diff --git a/api/v1alpha1/openapi_generated.go b/api/v1alpha1/openapi_generated.go index 4237a0c92ad..47e9f70bc67 100644 --- a/api/v1alpha1/openapi_generated.go +++ b/api/v1alpha1/openapi_generated.go @@ -6945,7 +6945,7 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_RuntimeTieredStoreLevel(ref com }, "low": { SchemaProps: spec.SchemaProps{ - Description: "Low is the ratio of low watermark of the tier (e.g., \"0.7\"). Eviction will continue until cache usage falls below this ratio. Type specifies the type of hostPath volume. Defaults to empty string (no validation). More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + Description: "Low is the ratio of low watermark of the tier (e.g., \"0.7\"). Eviction will continue until cache usage falls below this ratio.", Type: []string{"string"}, Format: "", }, diff --git a/pkg/common/cacheruntime.go b/pkg/common/cacheruntime.go index 56b02e69494..d25f807b1e0 100644 --- a/pkg/common/cacheruntime.go +++ b/pkg/common/cacheruntime.go @@ -134,7 +134,7 @@ func GetCacheRuntimeConfigConfigMapName(name string) string { // This config will be mounted into the worker container via ConfigMap. type TieredStoreLevelConfig struct { // MountPaths contains the mount paths inside the container for this tier - // For processMemory: empty array (no mount path) + // For processMemory: single element array with the mount path (e.g. ["/dev/shm"]) // For emptyDir: single element array with the mount path // For hostPath: array of mount paths corresponding to each host path MountPaths []string `json:"mountPaths,omitempty"` diff --git a/pkg/ddc/cache/engine/transform_tiered_store.go b/pkg/ddc/cache/engine/transform_tiered_store.go index 9069af25f1b..af4c06111d4 100644 --- a/pkg/ddc/cache/engine/transform_tiered_store.go +++ b/pkg/ddc/cache/engine/transform_tiered_store.go @@ -101,7 +101,7 @@ func (e *CacheEngine) handleProcessMemory(podSpec *corev1.PodSpec, container *co // add an memory emptyDir for /dev/shm in the container volumeName := fmt.Sprintf("tiered-store-level-%d-memory", levelIndex) - mountPath := GetEmptyDirTieredStoreMountPath(levelIndex) + mountPath := GetMemoryTieredStoreMountPath(levelIndex) volume := corev1.Volume{ Name: volumeName, VolumeSource: corev1.VolumeSource{ diff --git a/pkg/ddc/cache/engine/util.go b/pkg/ddc/cache/engine/util.go index a6650e97168..00dc52eb3ca 100644 --- a/pkg/ddc/cache/engine/util.go +++ b/pkg/ddc/cache/engine/util.go @@ -125,7 +125,7 @@ func GetEmptyDirTieredStoreMountPath(levelIndex int) string { return getTieredStoreMountPath(levelIndex, 0, "emptydir") } -// getTieredStoreMountPath generates the volume name for a tiered store medium +// getTieredStoreMountPath generates the mount path for a tiered store medium func getTieredStoreMountPath(levelIndex int, pathIndex int, mediumType string) string { return fmt.Sprintf("/etc/fluid/mount/tiered-store/level-%d-index-%d-%s", levelIndex, pathIndex, mediumType) } From 40cb1a06c33db1281363d5166a6ff62c05deaa13 Mon Sep 17 00:00:00 2001 From: xliuqq Date: Thu, 11 Jun 2026 22:11:37 +0800 Subject: [PATCH 4/7] update docs Signed-off-by: xliuqq --- .../dev/generic_cache_runtime_integration.md | 28 ++- .../en/userguide/cache_runtime_tieredstore.md | 221 ++++++++++++++++ .../dev/generic_cache_runtime_integration.md | 28 ++- .../zh/userguide/cache_runtime_tieredstore.md | 235 +++++++----------- .../engine/transform_tiered_store_test.go | 10 + 5 files changed, 359 insertions(+), 163 deletions(-) create mode 100644 docs/en/userguide/cache_runtime_tieredstore.md diff --git a/docs/en/dev/generic_cache_runtime_integration.md b/docs/en/dev/generic_cache_runtime_integration.md index 49d64d45ab8..003431dd84e 100644 --- a/docs/en/dev/generic_cache_runtime_integration.md +++ b/docs/en/dev/generic_cache_runtime_integration.md @@ -230,14 +230,11 @@ spec: key2: value2 replicas: 2 # worker tieredStore: - levels: # worker cache configuration - - quota: 40Gi - low: "0.5" + levels: # worker cache configuration + - emptyDir: + quota: 1Gi high: "0.8" - path: "/cache-data" - medium: - emptyDir: # Use tmpfs as cache medium - medium: Memory + low: "0.5" client: options: key1: value1 @@ -252,6 +249,8 @@ spec: ``` +For detailed configuration instructions on tiered storage, please refer to [CacheRuntime TieredStore Configuration Example](../userguide/cache_runtime_tieredstore.md). + ### Step 2.6 Confirm RuntimeConfig Provided by Fluid CacheRuntime for Components, Parse Parameters to Start Containers > You can modify the entryPoint script based on the native image, first parse RuntimeConfig, generate corresponding configuration files, and then start the container. > You can refer to the integration example in test/gha-e2e/curvine in the official repository. @@ -305,7 +304,20 @@ the `mounts`, `accessModes`, and `targetPath` fields in the JSON are all derived "replicas": 1, "service": { "name": "svc-curvine-demo-worker" - } + }, + "tieredStoreLevels": [ + { + "mountPaths": [ + "/etc/fluid/mount/tiered-store/level-0-index-0-emptydir" + ], + "mediumType": "HDD", + "quotas": [ + "1Gi" + ], + "high": "0.8", + "low": "0.5" + } + ] }, "client": { "enabled": true, diff --git a/docs/en/userguide/cache_runtime_tieredstore.md b/docs/en/userguide/cache_runtime_tieredstore.md new file mode 100644 index 00000000000..a6582159e7f --- /dev/null +++ b/docs/en/userguide/cache_runtime_tieredstore.md @@ -0,0 +1,221 @@ +# CacheRuntime TieredStore Configuration Example + +This document demonstrates how to configure tiered storage (TieredStore) for CacheRuntime. + +## Overview + +CacheRuntime supports three types of storage media: +1. **ProcessMemory**: Uses process memory as cache storage +2. **EmptyDir**: Uses Kubernetes EmptyDir Volume as cache storage +3. **HostPath**: Uses Kubernetes HostPath Volume as cache storage + +The system automatically calculates `mediumType` based on the configuration for cache strategy optimization: +- **MEM**: Memory medium +- **HDD**: Hard disk medium + +## Example 1: Using ProcessMemory + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - processMemory: + quota: 8Gi + high: "0.95" + low: "0.7" +``` + +**Notes:** +- `processMemory.quota` sets the memory quota, which will be automatically added to the container's resource requests/limits +- `high` and `low` are watermark configurations used to control cache eviction + +## Example 2: Using EmptyDir Volume (Disk) + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - emptyDir: + quota: 100Gi + medium: "" # Use disk space, set to "Memory" for tmpfs + high: "0.95" + low: "0.7" +``` + +**Notes:** +- `emptyDir` creates a temporary directory with the same lifecycle as the Pod +- `quota` is set as the EmptyDir's sizeLimit +- Cache data will be lost after Pod restart + +## Example 3: Using HostPath Volume (Single Path) + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - hostPath: + paths: + - /data/cache + quotas: + - 100Gi + type: DirectoryOrCreate + high: "0.95" + low: "0.7" +``` + +**Notes:** +- `hostPath` uses a persistent directory on the node +- `paths` specifies the list of cache paths +- `quotas` specifies the quota for each path +- Cache data persists after Pod restart +- Ensure proper directory permissions on the node + +## Example 4: Multi-tier Storage + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + # Level 1: High-speed memory cache (mediumType automatically set to MEM) + - processMemory: + quota: 4Gi + high: "0.95" + low: "0.7" + + # Level 2: Large-capacity disk cache (mediumType automatically set to HDD) + - emptyDir: + quota: 200Gi + high: "0.90" + low: "0.6" +``` + +**Notes:** +- Multiple tiers can be configured, ordered from highest to lowest priority +- Each tier can have different media types, quotas, and watermarks +- The system prioritizes higher tiers + +## Example 5: HostPath Multi-path Configuration + +```yaml +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: my-cache + namespace: default +spec: + runtimeClassName: my-runtime-class + worker: + tieredStore: + levels: + - hostPath: + paths: + - /data/cache1 + - /data/cache2 + quotas: + - 100Gi + - 200Gi + type: DirectoryOrCreate + high: "0.95" + low: "0.7" +``` + +**Notes:** +- Only HostPath type supports multi-path configuration +- The lengths of `paths` and `quotas` arrays must match +- Total quota = 100Gi + 200Gi = 300Gi + +## Implementation Details + +### mediumType Auto Calculation Logic + +`mediumType` is automatically calculated by the system based on the configuration, no manual configuration required: + +| Configuration Type | mediumType Value | Description | +|-------------------|------------------|-------------| +| `processMemory` | MEM | Memory medium | +| `emptyDir` (medium="Memory") | MEM | Uses tmpfs memory | +| `emptyDir` (medium="") | HDD | Uses node's default storage (disk) | +| `hostPath` | HDD | Hard disk medium | + +### ProcessMemory Handling + +When using `processMemory`: +1. Adds `quota` to the container's `resources.requests.memory` and `resources.limits.memory` +2. Accumulates if the container already has other resource requests +3. `mediumType` is automatically set to MEM + +### EmptyDir Handling + +When using `emptyDir`: +1. Creates an EmptyDir Volume +2. Volume name format: `tieredstore-level{N}` +3. `quota` is set as EmptyDir's `sizeLimit` +4. If `medium` is set to `"Memory"`, uses tmpfs, `mediumType` is MEM +5. If `medium` is empty, uses node's default storage, `mediumType` is HDD + +### HostPath Handling + +When using `hostPath`: +1. Creates an independent HostPath Volume for each path +2. Volume name format: `tieredstore-level{N}-path{M}` +3. `paths` list specifies directory paths on the node +4. `quotas` list specifies quotas for each path (for reference only, not enforced) +5. `type` specifies the hostPath type (e.g., `DirectoryOrCreate`) +6. `mediumType` is automatically set to HDD + +### Important Notes on Multi-path Configuration + +According to the latest API definition, **only HostPath type supports multi-path configuration**: + +| Media Type | Multi-path Support | Quota Enforcement | Description | +|------------|-------------------|------------------|-------------| +| **ProcessMemory** | ❌ Not supported | Via container memory limit | Single quota configuration | +| **EmptyDir** | ❌ Not supported | Via `sizeLimit` | Single quota configuration | +| **HostPath** | ✅ Supported | For reference only (requires external mechanism) | Multiple paths configurable | + +**HostPath Multi-path Details:** +- ✅ Each path creates an independent HostPath volume +- ✅ `paths` and `quotas` arrays must have matching lengths +- ⚠️ `quotas` are for reference only, external mechanisms (e.g., XFS quota, Cgroup) are required for enforcement +- 💡 Main purpose of multi-path: Expand total cache capacity or distribute across multiple disks + +## Notes + +1. **Quota Units**: Supports Kubernetes standard units (Ki, Mi, Gi, Ti, etc.) +2. **Watermarks**: `high` and `low` are decimals between 0-1, representing usage percentage +3. **Media Selection**: Each tier can only select one media type (ProcessMemory, EmptyDir, or HostPath) +4. **Multi-path Support**: Only HostPath type supports multi-path configuration; ProcessMemory and EmptyDir only support single path +5. **HostPath Path Requirements**: `paths` and `quotas` arrays must have matching lengths +6. **mediumType**: Automatically calculated by the system, no manual configuration needed, only supports MEM and HDD values +7. **Quota Enforcement Mechanism**: + - ProcessMemory: Enforced via container memory limit + - EmptyDir: Enforced via `sizeLimit` + - HostPath: `quotas` are for reference only, requires external mechanisms (e.g., XFS project quota, Cgroup) \ No newline at end of file diff --git a/docs/zh/dev/generic_cache_runtime_integration.md b/docs/zh/dev/generic_cache_runtime_integration.md index 20c84833375..1aa7ff8cd9f 100644 --- a/docs/zh/dev/generic_cache_runtime_integration.md +++ b/docs/zh/dev/generic_cache_runtime_integration.md @@ -230,14 +230,11 @@ spec: key2: value2 replicas: 2 #worker tieredStore: - levels: #worker缓存配置 - - quota: 40Gi - low: "0.5" + levels: #worker缓存配置 + - emptyDir: + quota: 1Gi high: "0.8" - path: "/cache-data" - medium: - emptyDir: #使用tmpfs作为缓存介质 - medium: Memory + low: "0.5" client: options: key1: value1 @@ -252,6 +249,8 @@ spec: ``` +关于分层存储的详细配置说明，请参考 [CacheRuntime TieredStore 配置示例](../userguide/cache_runtime_tieredstore.md)。 + ### 步骤2.6 确认 Fluid CacheRuntime为组件提供的 RuntimeConfig，进行参数解析启动容器 > 可以基于原生镜像改造 entryPoint 脚本，先解析 RuntimeConfig，并生成对应的配置文件，后再启动容器。 > 可以参考官方仓库中的 test/gha-e2e/curvine 的集成示例。 @@ -304,7 +303,20 @@ spec: "replicas": 1, "service": { "name": "svc-curvine-demo-worker" - } + }, + "tieredStoreLevels": [ + { + "mountPaths": [ + "/etc/fluid/mount/tiered-store/level-0-index-0-emptydir" + ], + "mediumType": "HDD", + "quotas": [ + "1Gi" + ], + "high": "0.8", + "low": "0.5" + } + ] }, "client": { "enabled": true, diff --git a/docs/zh/userguide/cache_runtime_tieredstore.md b/docs/zh/userguide/cache_runtime_tieredstore.md index 06a987784af..e5f32624889 100644 --- a/docs/zh/userguide/cache_runtime_tieredstore.md +++ b/docs/zh/userguide/cache_runtime_tieredstore.md @@ -4,9 +4,14 @@ ## 概述 -CacheRuntime 支持两种类型的存储介质： +CacheRuntime 支持三种类型的存储介质： 1. **ProcessMemory**：使用进程内存作为缓存存储 -2. **Volume**：使用 Kubernetes Volume 作为缓存存储（支持 hostPath、emptyDir、ephemeral） +2. **EmptyDir**：使用 Kubernetes EmptyDir Volume 作为缓存存储 +3. **HostPath**：使用 Kubernetes HostPath Volume 作为缓存存储 + +系统会根据配置自动计算 `mediumType`，用于缓存策略优化： +- **MEM**：内存介质 +- **HDD**：硬盘介质 ## 示例 1：使用 ProcessMemory @@ -21,23 +26,17 @@ spec: worker: tieredStore: levels: - - medium: - processMemory: {} - path: - - /dev/shm - quota: - - 8Gi + - processMemory: + quota: 8Gi high: "0.95" low: "0.7" ``` **说明：** -- `processMemory: {}` 表示使用进程内存 -- `path` 指定缓存路径（对于内存，通常是 /dev/shm） -- `quota` 设置内存配额，会自动添加到容器的 resource requests/limits +- `processMemory.quota` 设置内存配额，会自动添加到容器的 resource requests/limits - `high` 和 `low` 是水位线配置，用于控制缓存驱逐 -## 示例 2：使用 EmptyDir Volume +## 示例 2：使用 EmptyDir Volume（磁盘） ```yaml apiVersion: data.fluid.io/v1alpha1 @@ -50,14 +49,9 @@ spec: worker: tieredStore: levels: - - medium: - volume: - emptyDir: - medium: "" # 使用磁盘空间，如果要使用内存设置为 "Memory" - path: - - /mnt/cache - quota: - - 100Gi + - emptyDir: + quota: 100Gi + medium: "" # 使用磁盘空间，如果要使用内存设置为 "Memory" high: "0.95" low: "0.7" ``` @@ -67,7 +61,7 @@ spec: - `quota` 会设置为 EmptyDir 的 sizeLimit - 缓存数据会在 Pod 重启后丢失 -## 示例 3：使用 HostPath Volume +## 示例 3：使用 HostPath Volume（单路径） ```yaml apiVersion: data.fluid.io/v1alpha1 @@ -80,19 +74,20 @@ spec: worker: tieredStore: levels: - - medium: - volume: - hostPath: - path: /data/cache - type: DirectoryOrCreate - path: - - /mnt/cache + - hostPath: + paths: + - /data/cache + quotas: + - 100Gi + type: DirectoryOrCreate high: "0.95" low: "0.7" ``` **说明：** - `hostPath` 使用节点上的持久化目录 +- `paths` 指定缓存路径列表 +- `quotas` 指定每个路径的配额 - 缓存数据在 Pod 重启后仍然保留 - 需要确保节点上有相应的目录权限 @@ -109,24 +104,15 @@ spec: worker: tieredStore: levels: - # 第一层：高速内存缓存 - - medium: - processMemory: {} - path: - - /dev/shm - quota: - - 4Gi + # 第一层：高速内存缓存（mediumType 自动设置为 MEM） + - processMemory: + quota: 4Gi high: "0.95" low: "0.7" - # 第二层：大容量磁盘缓存 - - medium: - volume: - emptyDir: {} - path: - - /mnt/ssd - quota: - - 200Gi + # 第二层：大容量磁盘缓存（mediumType 自动设置为 HDD） + - emptyDir: + quota: 200Gi high: "0.90" low: "0.6" ``` @@ -136,7 +122,7 @@ spec: - 每层可以有不同的介质类型、配额和水位线 - 系统会优先使用高层级的存储 -## 示例 5：多路径配置 +## 示例 5：HostPath 多路径配置 ```yaml apiVersion: data.fluid.io/v1alpha1 @@ -149,132 +135,87 @@ spec: worker: tieredStore: levels: - - medium: - processMemory: {} - path: - - /dev/shm/cache1 - - /dev/shm/cache2 - quota: - - 2Gi - - 3Gi + - hostPath: + paths: + - /data/cache1 + - /data/cache2 + quotas: + - 100Gi + - 200Gi + type: DirectoryOrCreate high: "0.95" low: "0.7" ``` **说明：** -- 可以为同一层级指定多个路径 -- 每个路径可以有独立的配额 -- 总内存配额 = 2Gi + 3Gi = 5Gi +- 只有 HostPath 类型支持多路径配置 +- `paths` 和 `quotas` 数组长度必须一致 +- 总配额 = 100Gi + 200Gi = 300Gi -## 示例 6：使用 Ephemeral Volume +## 实现细节 -```yaml -apiVersion: data.fluid.io/v1alpha1 -kind: CacheRuntime -metadata: - name: my-cache - namespace: default -spec: - runtimeClassName: my-runtime-class - worker: - tieredStore: - levels: - - medium: - volume: - ephemeral: - volumeClaimTemplate: - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: "fast-ssd" - path: - - /mnt/cache - quota: - - 100Gi - high: "0.95" - low: "0.7" -``` +### mediumType 自动计算逻辑 -**说明：** -- `ephemeral` 会创建临时的 PVC，生命周期与 Pod 相同 -- `quota` 会设置为 PVC 的 storage request -- 可以使用不同的 storageClassName 来获得不同的存储性能 -- 缓存数据会在 Pod 删除后丢失，但比 EmptyDir 更灵活 +`mediumType` 是由系统根据配置自动计算的，用户无需手动配置： -## 实现细节 +| 配置类型 | mediumType 值 | 说明 | +|---------|--------------|------| +| `processMemory` | MEM | 内存介质 | +| `emptyDir`（medium="Memory"） | MEM | 使用 tmpfs 内存 | +| `emptyDir`（medium=""） | HDD | 使用节点默认存储（磁盘） | +| `hostPath` | HDD | 硬盘介质 | ### ProcessMemory 处理 当使用 `processMemory` 时： -1. 系统会计算所有路径的配额总和 -2. 将总配额添加到容器的 `resources.requests.memory` 和 `resources.limits.memory` -3. 如果容器已有其他资源请求，会累加而不是覆盖 +1. 将 `quota` 添加到容器的 `resources.requests.memory` 和 `resources.limits.memory` +2. 如果容器已有其他资源请求，会累加而不是覆盖 +3. `mediumType` 自动设置为 MEM + +### EmptyDir 处理 + +当使用 `emptyDir` 时： +1. 创建一个 EmptyDir Volume +2. Volume 名称格式：`tieredstore-level{N}` +3. `quota` 设置为 EmptyDir 的 `sizeLimit` +4. 如果 `medium` 设置为 `"Memory"`，则使用 tmpfs，`mediumType` 为 MEM +5. 如果 `medium` 为空，则使用节点默认存储，`mediumType` 为 HDD -### Volume 处理 +### HostPath 处理 -当使用 `volume` 时： -1. 为每个路径创建一个独立的 Volume +当使用 `hostPath` 时： +1. 为每个路径创建一个独立的 HostPath Volume 2. Volume 名称格式：`tieredstore-level{N}-path{M}` -3. 根据 Volume 类型（hostPath/emptyDir/ephemeral）创建对应的 VolumeSource -4. 为容器添加对应的 VolumeMount -5. 配额处理方式因类型而异： - - **EmptyDir**：quota 设置为 `sizeLimit` - - **Ephemeral**：quota 设置为 `volumeClaimTemplate.spec.resources.requests.storage` - - **HostPath**：quota **仅供参考**，不会强制限制（需要依赖外部机制如 XFS quota） +3. `paths` 列表指定节点上的目录路径 +4. `quotas` 列表指定每个路径的配额（仅供参考，不会强制限制） +5. `type` 指定 hostPath 类型（如 `DirectoryOrCreate`） +6. `mediumType` 自动设置为 HDD ### 多路径配置的重要说明 -**不同 Volume 类型对多路径的支持程度：** - -| Volume 类型 | 多路径合理性 | Quota 是否生效 | 物理隔离 | 建议 | -|------------|------------|--------------|---------|------| -| **EmptyDir** | ✅ 合理 | ✅ 是（通过 sizeLimit） | ✅ 独立 tmpfs/disk | 推荐使用 | -| **Ephemeral** | ✅ 合理 | ✅ 是（通过 PVC storage request） | ✅ 独立 PVC | 推荐使用 | -| **HostPath** | ❌ 不推荐 | ❌ 否（共享节点存储） | ❌ 无隔离 | 建议使用单路径 | -| **ProcessMemory** | ⚠️ 有限 | ⚠️ 部分（累加到容器 limit） | ❌ 无隔离 | 谨慎使用多路径 | - -**详细说明：** - -1. **EmptyDir + 多路径**： - - ✅ 每个路径创建独立的 EmptyDir volume - - ✅ 每个 volume 有独立的 `sizeLimit` - - ✅ 真正的物理隔离和配额控制 - - 示例：可以使用多个磁盘分区或 tmpfs - -2. **Ephemeral + 多路径**： - - ✅ 每个路径创建独立的 PVC - - ✅ 每个 PVC 可以有独立的 storage request（通过 quota 设置） - - ⚠️ **所有 PVC 共享相同的 volumeClaimTemplate 配置**（包括 storageClassName、accessModes 等） - - ⚠️ **无法在同一个 Level 内混合使用不同的 storage class** - - 💡 多路径的主要用途：**扩展同一类型存储的总容量** - - 示例：使用多个 SSD PVC，每个 100Gi，总容量 200Gi - - 📌 如需使用不同类型的存储（SSD + HDD），应配置**不同的 Level** - -3. **HostPath + 多路径**： - - ❌ 多个路径可能指向同一节点的存储空间 - - ❌ quota 无法直接生效，只是配置元数据 - - ❌ 没有真正的配额限制和物理隔离 - - ⚠️ 如果需要配额控制，请使用操作系统级别的 quota 机制 - - **建议**：HostPath 场景下只配置单个路径 - -4. **ProcessMemory + 多路径**： - - ⚠️ 所有路径的 quota 会累加到容器的 memory requests/limits - - ⚠️ 路径之间没有物理隔离（都在 /dev/shm 下） - - ⚠️ 应用程序可以随意使用总内存，不受单个路径 quota 限制 - - **建议**：除非有特殊的逻辑分区需求，否则使用单路径即可 +根据最新的 API 定义，**只有 HostPath 类型支持多路径配置**： + +| 介质类型 | 多路径支持 | Quota 生效方式 | 说明 | +|---------|-----------|---------------|------| +| **ProcessMemory** | ❌ 不支持 | 通过容器 memory limit | 单配额配置 | +| **EmptyDir** | ❌ 不支持 | 通过 `sizeLimit` | 单配额配置 | +| **HostPath** | ✅ 支持 | 仅供参考（需外部机制） | 可配置多个路径 | + +**HostPath 多路径详细说明：** +- ✅ 每个路径创建独立的 HostPath volume +- ✅ `paths` 和 `quotas` 数组长度必须一致 +- ⚠️ `quotas` 仅供参考，需要依赖外部机制（如 XFS quota、Cgroup 等）进行强制限制 +- 💡 多路径的主要用途：扩展缓存容量或分布到多个磁盘 ## 注意事项 1. **配额单位**：支持 Kubernetes 标准单位（Ki, Mi, Gi, Ti 等） 2. **水位线**：`high` 和 `low` 是 0-1 之间的小数，表示使用率的百分比 -3. **路径要求**：`path` 数组和 `quota` 数组长度必须一致 -4. **介质选择**：每个层级只能选择一种介质类型（ProcessMemory 或 Volume） -5. **Volume 限制**：每个层级的 Volume 只能选择一种类型（hostPath、emptyDir 或 ephemeral） -6. **多路径建议**： - - EmptyDir 和 Ephemeral 类型推荐使用多路径，可以实现真正的配额控制和物理隔离 - - HostPath 类型建议使用单路径，因为 quota 无法生效 - - ProcessMemory 类型谨慎使用多路径，只有逻辑分区意义，无物理隔离 +3. **介质选择**：每个层级只能选择一种介质类型（ProcessMemory、EmptyDir 或 HostPath） +4. **多路径支持**：只有 HostPath 类型支持多路径配置，ProcessMemory 和 EmptyDir 仅支持单路径 +5. **HostPath 路径要求**：`paths` 和 `quotas` 数组长度必须一致 +6. **mediumType**：由系统自动计算，无需用户配置，仅支持 MEM 和 HDD 两种值 7. **配额生效机制**： + - ProcessMemory：通过容器 memory limit 强制限制 - EmptyDir：通过 `sizeLimit` 强制限制 - - Ephemeral：通过 PVC 的 storage request 限制 - - HostPath：需要依赖外部机制（如 XFS project quota、Cgroup 等） - - ProcessMemory：通过容器 memory limit 限制总量，但路径间无隔离 + - HostPath：`quotas` 仅供参考，需要依赖外部机制（如 XFS project quota、Cgroup 等） \ No newline at end of file diff --git a/pkg/ddc/cache/engine/transform_tiered_store_test.go b/pkg/ddc/cache/engine/transform_tiered_store_test.go index d7f27eb1df2..bbad70f8a3f 100644 --- a/pkg/ddc/cache/engine/transform_tiered_store_test.go +++ b/pkg/ddc/cache/engine/transform_tiered_store_test.go @@ -113,6 +113,16 @@ var _ = Describe("CacheEngine TransformRuntimeTieredStore Tests", Label("pkg.ddc memLimit := podSpec.Containers[0].Resources.Limits[corev1.ResourceMemory] expectedLimit := resource.MustParse("8Gi") // 4Gi + 4Gi Expect(memLimit.Cmp(expectedLimit)).To(Equal(0)) + + // Verify volume and volume mount are created for ProcessMemory + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("tiered-store-level-0-memory")) + Expect(podSpec.Volumes[0].EmptyDir).NotTo(BeNil()) + Expect(podSpec.Volumes[0].EmptyDir.Medium).To(Equal(corev1.StorageMediumMemory)) + + // Verify volume mount path is set to /dev/shm (from GetMemoryTieredStoreMountPath) + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/dev/shm")) }) It("should return error when quota is zero", func() { From 19f8b3f7660cc74dd22178483f775e25fe938569 Mon Sep 17 00:00:00 2001 From: xliuqq Date: Fri, 12 Jun 2026 23:52:27 +0800 Subject: [PATCH 5/7] fix reviewers Signed-off-by: xliuqq --- .../en/userguide/cache_runtime_tieredstore.md | 25 ++++++++++++------- .../zh/userguide/cache_runtime_tieredstore.md | 25 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/docs/en/userguide/cache_runtime_tieredstore.md b/docs/en/userguide/cache_runtime_tieredstore.md index a6582159e7f..c6e27c4476f 100644 --- a/docs/en/userguide/cache_runtime_tieredstore.md +++ b/docs/en/userguide/cache_runtime_tieredstore.md @@ -171,25 +171,32 @@ When using `processMemory`: 1. Adds `quota` to the container's `resources.requests.memory` and `resources.limits.memory` 2. Accumulates if the container already has other resource requests 3. `mediumType` is automatically set to MEM +4. Creates an EmptyDir Volume with memory medium +5. Volume name format: `tiered-store-level-{N}-memory` +6. Mount path in container: `/dev/shm` ### EmptyDir Handling When using `emptyDir`: 1. Creates an EmptyDir Volume -2. Volume name format: `tieredstore-level{N}` -3. `quota` is set as EmptyDir's `sizeLimit` -4. If `medium` is set to `"Memory"`, uses tmpfs, `mediumType` is MEM -5. If `medium` is empty, uses node's default storage, `mediumType` is HDD +2. Volume name format: `tiered-store-level-{N}-index-{M}` (M is always 0 for EmptyDir) +3. Mount path in container: `/etc/fluid/mount/tiered-store/level-{N}-index-{M}-emptydir` +4. `quota` is set as EmptyDir's `sizeLimit` +5. If `medium` is set to `"Memory"`, uses tmpfs, `mediumType` is MEM +6. If `medium` is empty, uses node's default storage, `mediumType` is HDD ### HostPath Handling When using `hostPath`: 1. Creates an independent HostPath Volume for each path -2. Volume name format: `tieredstore-level{N}-path{M}` -3. `paths` list specifies directory paths on the node -4. `quotas` list specifies quotas for each path (for reference only, not enforced) -5. `type` specifies the hostPath type (e.g., `DirectoryOrCreate`) -6. `mediumType` is automatically set to HDD +2. Volume name format: `tiered-store-level-{N}-index-{M}` +3. Mount path in container: `/etc/fluid/mount/tiered-store/level-{N}-index-{M}-hostpath` +4. `paths` list specifies directory paths on the node +5. `quotas` list specifies quotas for each path (for reference only, not enforced) +6. `type` specifies the hostPath type (e.g., `DirectoryOrCreate`) +7. `mediumType` is automatically set to HDD + +> **Note:** Volume names are implementation details and may change. Users should not depend on specific volume name formats. ### Important Notes on Multi-path Configuration diff --git a/docs/zh/userguide/cache_runtime_tieredstore.md b/docs/zh/userguide/cache_runtime_tieredstore.md index e5f32624889..cc3015306ae 100644 --- a/docs/zh/userguide/cache_runtime_tieredstore.md +++ b/docs/zh/userguide/cache_runtime_tieredstore.md @@ -171,25 +171,32 @@ spec: 1. 将 `quota` 添加到容器的 `resources.requests.memory` 和 `resources.limits.memory` 2. 如果容器已有其他资源请求，会累加而不是覆盖 3. `mediumType` 自动设置为 MEM +4. 创建一个使用内存介质的 EmptyDir Volume +5. Volume 名称格式：`tiered-store-level-{N}-memory` +6. 容器内挂载路径：`/dev/shm` ### EmptyDir 处理 当使用 `emptyDir` 时： 1. 创建一个 EmptyDir Volume -2. Volume 名称格式：`tieredstore-level{N}` -3. `quota` 设置为 EmptyDir 的 `sizeLimit` -4. 如果 `medium` 设置为 `"Memory"`，则使用 tmpfs，`mediumType` 为 MEM -5. 如果 `medium` 为空，则使用节点默认存储，`mediumType` 为 HDD +2. Volume 名称格式：`tiered-store-level-{N}-index-{M}`（EmptyDir 的 M 始终为 0） +3. 容器内挂载路径：`/etc/fluid/mount/tiered-store/level-{N}-index-{M}-emptydir` +4. `quota` 设置为 EmptyDir 的 `sizeLimit` +5. 如果 `medium` 设置为 `"Memory"`，则使用 tmpfs，`mediumType` 为 MEM +6. 如果 `medium` 为空，则使用节点默认存储，`mediumType` 为 HDD ### HostPath 处理 当使用 `hostPath` 时： 1. 为每个路径创建一个独立的 HostPath Volume -2. Volume 名称格式：`tieredstore-level{N}-path{M}` -3. `paths` 列表指定节点上的目录路径 -4. `quotas` 列表指定每个路径的配额（仅供参考，不会强制限制） -5. `type` 指定 hostPath 类型（如 `DirectoryOrCreate`） -6. `mediumType` 自动设置为 HDD +2. Volume 名称格式：`tiered-store-level-{N}-index-{M}` +3. 容器内挂载路径：`/etc/fluid/mount/tiered-store/level-{N}-index-{M}-hostpath` +4. `paths` 列表指定节点上的目录路径 +5. `quotas` 列表指定每个路径的配额（仅供参考，不会强制限制） +6. `type` 指定 hostPath 类型（如 `DirectoryOrCreate`） +7. `mediumType` 自动设置为 HDD + +> **注意**：Volume 名称属于实现细节，可能会发生变化。用户不应依赖特定的 Volume 名称格式。 ### 多路径配置的重要说明 From ccff777c83021e7c47f010b58be6ffd000b311f8 Mon Sep 17 00:00:00 2001 From: xliuqq Date: Sun, 14 Jun 2026 17:22:10 +0800 Subject: [PATCH 6/7] fix reviews Signed-off-by: xliuqq --- api/v1alpha1/zz_generated.deepcopy.go | 114 ++++++++---------- .../en/userguide/cache_runtime_tieredstore.md | 2 +- .../zh/userguide/cache_runtime_tieredstore.md | 2 +- 3 files changed, 52 insertions(+), 66 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d5a812ef715..f66b3e216bd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,20 +1,5 @@ //go:build !ignore_autogenerated -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -1581,6 +1566,22 @@ func (in *EFCRuntimeSpec) DeepCopy() *EFCRuntimeSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EmptyDirMediumSource) DeepCopyInto(out *EmptyDirMediumSource) { + *out = *in + out.Quota = in.Quota.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyDirMediumSource. +func (in *EmptyDirMediumSource) DeepCopy() *EmptyDirMediumSource { + if in == nil { + return nil + } + out := new(EmptyDirMediumSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EncryptOption) DeepCopyInto(out *EncryptOption) { *out = *in @@ -1755,6 +1756,38 @@ func (in *HeadlessRuntimeComponentService) DeepCopy() *HeadlessRuntimeComponentS return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostPathMediumSource) DeepCopyInto(out *HostPathMediumSource) { + *out = *in + if in.Paths != nil { + in, out := &in.Paths, &out.Paths + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Quotas != nil { + in, out := &in.Quotas, &out.Quotas + *out = make([]resource.Quantity, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(v1.HostPathType) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMediumSource. +func (in *HostPathMediumSource) DeepCopy() *HostPathMediumSource { + if in == nil { + return nil + } + out := new(HostPathMediumSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InitFuseSpec) DeepCopyInto(out *InitFuseSpec) { *out = *in @@ -2519,6 +2552,7 @@ func (in *Prefer) DeepCopy() *Prefer { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProcessMemoryMediumSource) DeepCopyInto(out *ProcessMemoryMediumSource) { *out = *in + out.Quota = in.Quota.DeepCopy() } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProcessMemoryMediumSource. @@ -2531,54 +2565,6 @@ func (in *ProcessMemoryMediumSource) DeepCopy() *ProcessMemoryMediumSource { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EmptyDirMediumSource) DeepCopyInto(out *EmptyDirMediumSource) { - *out = *in - out.Quota = in.Quota.DeepCopy() -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyDirMediumSource. -func (in *EmptyDirMediumSource) DeepCopy() *EmptyDirMediumSource { - if in == nil { - return nil - } - out := new(EmptyDirMediumSource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HostPathMediumSource) DeepCopyInto(out *HostPathMediumSource) { - *out = *in - if in.Paths != nil { - in, out := &in.Paths, &out.Paths - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Quotas != nil { - in, out := &in.Quotas, &out.Quotas - *out = make([]resource.Quantity, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Type != nil { - in, out := &in.Type, &out.Type - *out = new(v1.HostPathType) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMediumSource. -func (in *HostPathMediumSource) DeepCopy() *HostPathMediumSource { - if in == nil { - return nil - } - out := new(HostPathMediumSource) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Processor) DeepCopyInto(out *Processor) { *out = *in @@ -2931,7 +2917,7 @@ func (in *RuntimeTieredStoreLevel) DeepCopyInto(out *RuntimeTieredStoreLevel) { if in.ProcessMemory != nil { in, out := &in.ProcessMemory, &out.ProcessMemory *out = new(ProcessMemoryMediumSource) - **out = **in + (*in).DeepCopyInto(*out) } if in.EmptyDir != nil { in, out := &in.EmptyDir, &out.EmptyDir diff --git a/docs/en/userguide/cache_runtime_tieredstore.md b/docs/en/userguide/cache_runtime_tieredstore.md index c6e27c4476f..6cff99bdd7f 100644 --- a/docs/en/userguide/cache_runtime_tieredstore.md +++ b/docs/en/userguide/cache_runtime_tieredstore.md @@ -225,4 +225,4 @@ According to the latest API definition, **only HostPath type supports multi-path 7. **Quota Enforcement Mechanism**: - ProcessMemory: Enforced via container memory limit - EmptyDir: Enforced via `sizeLimit` - - HostPath: `quotas` are for reference only, requires external mechanisms (e.g., XFS project quota, Cgroup) \ No newline at end of file + - HostPath: `quotas` are for reference only, requires external mechanisms (e.g., XFS project quota, Cgroup) diff --git a/docs/zh/userguide/cache_runtime_tieredstore.md b/docs/zh/userguide/cache_runtime_tieredstore.md index cc3015306ae..bc8a05ee71a 100644 --- a/docs/zh/userguide/cache_runtime_tieredstore.md +++ b/docs/zh/userguide/cache_runtime_tieredstore.md @@ -225,4 +225,4 @@ spec: 7. **配额生效机制**： - ProcessMemory：通过容器 memory limit 强制限制 - EmptyDir：通过 `sizeLimit` 强制限制 - - HostPath：`quotas` 仅供参考，需要依赖外部机制（如 XFS project quota、Cgroup 等） \ No newline at end of file + - HostPath：`quotas` 仅供参考，需要依赖外部机制（如 XFS project quota、Cgroup 等） From ba09a5d7969b49604c888ed4424b7c476e69966c Mon Sep 17 00:00:00 2001 From: xliuqq Date: Mon, 15 Jun 2026 21:37:15 +0800 Subject: [PATCH 7/7] use qutoa deepcopy and check tiered store then set Signed-off-by: xliuqq --- .../cache/engine/transform_tiered_store.go | 48 ++++++++++++------- .../engine/transform_tiered_store_test.go | 21 ++++++++ 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/pkg/ddc/cache/engine/transform_tiered_store.go b/pkg/ddc/cache/engine/transform_tiered_store.go index af4c06111d4..ac451b45ee5 100644 --- a/pkg/ddc/cache/engine/transform_tiered_store.go +++ b/pkg/ddc/cache/engine/transform_tiered_store.go @@ -36,15 +36,36 @@ func (e *CacheEngine) TransformRuntimeTieredStore(tieredStore *datav1alpha1.Runt container := &podSpec.Containers[0] - // Process each tier level - for i, level := range tieredStore.Levels { + // validate then set + memoryLevelCount := 0 + for idx, level := range tieredStore.Levels { // order: memory, host path, empty. only one can be specified per level mediaCount := 0 + if level.ProcessMemory != nil { + mediaCount++ + memoryLevelCount++ + if memoryLevelCount > 1 { + return fmt.Errorf("RuntimeTieredStore should have only one ProcessMemoryMediumSource for all levels") + } + } + + if level.HostPath != nil { + mediaCount++ + } + if level.EmptyDir != nil { + mediaCount++ + } + if mediaCount > 1 { + return fmt.Errorf("only one storage medium can be specified per level at index %d, but found %d", idx, mediaCount) + } + } + + // Process each tier level + for idx, level := range tieredStore.Levels { // Process memory: add resource requests and limits if level.ProcessMemory != nil { - mediaCount++ - err := e.handleProcessMemory(podSpec, container, level.ProcessMemory, i) + err := e.handleProcessMemory(podSpec, container, level.ProcessMemory, idx) if err != nil { return err } @@ -52,8 +73,7 @@ func (e *CacheEngine) TransformRuntimeTieredStore(tieredStore *datav1alpha1.Runt // Volume-based storage: create volumes and volume mounts if level.HostPath != nil { - mediaCount++ - err := e.handleHostPath(podSpec, container, level.HostPath, i) + err := e.handleHostPath(podSpec, container, level.HostPath, idx) if err != nil { return err } @@ -61,16 +81,11 @@ func (e *CacheEngine) TransformRuntimeTieredStore(tieredStore *datav1alpha1.Runt // EmptyDir: add volume and volume mount if level.EmptyDir != nil { - mediaCount++ - err := e.handleEmptyDir(podSpec, container, level.EmptyDir, i) + err := e.handleEmptyDir(podSpec, container, level.EmptyDir, idx) if err != nil { return err } } - - if mediaCount > 1 { - return fmt.Errorf("only one storage medium can be specified per level at index %d, but found %d", i, mediaCount) - } } return nil @@ -83,7 +98,7 @@ func (e *CacheEngine) handleProcessMemory(podSpec *corev1.PodSpec, container *co } // Calculate total memory quota across all paths - totalQuota := memoryMediumSource.Quota + totalQuota := memoryMediumSource.Quota.DeepCopy() // add totalQuota to memory resources only when memory is restricted. if container.Resources.Requests != nil { @@ -172,12 +187,13 @@ func (e *CacheEngine) handleEmptyDir(podSpec *corev1.PodSpec, container *corev1. volumeName := fmt.Sprintf("tiered-store-level-%d-index-%d", levelIndex, 0) mountPath := GetEmptyDirTieredStoreMountPath(levelIndex) + quota := emptyDirMediumSource.Quota.DeepCopy() volume := corev1.Volume{ Name: volumeName, VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: emptyDirMediumSource.Medium, - SizeLimit: &emptyDirMediumSource.Quota, + SizeLimit: "a, }, }, } @@ -199,13 +215,13 @@ func (e *CacheEngine) handleEmptyDir(podSpec *corev1.PodSpec, container *corev1. // If no memory resources are set, the container is unconstrained and we don't need to add if container.Resources.Requests != nil { if currentRequest, exists := container.Resources.Requests[corev1.ResourceMemory]; exists && !currentRequest.IsZero() { - currentRequest.Add(emptyDirMediumSource.Quota) + currentRequest.Add(quota) container.Resources.Requests[corev1.ResourceMemory] = currentRequest } } if container.Resources.Limits != nil { if currentLimit, exists := container.Resources.Limits[corev1.ResourceMemory]; exists && !currentLimit.IsZero() { - currentLimit.Add(emptyDirMediumSource.Quota) + currentLimit.Add(quota) container.Resources.Limits[corev1.ResourceMemory] = currentLimit } } diff --git a/pkg/ddc/cache/engine/transform_tiered_store_test.go b/pkg/ddc/cache/engine/transform_tiered_store_test.go index bbad70f8a3f..44099d9ae2e 100644 --- a/pkg/ddc/cache/engine/transform_tiered_store_test.go +++ b/pkg/ddc/cache/engine/transform_tiered_store_test.go @@ -163,6 +163,27 @@ var _ = Describe("CacheEngine TransformRuntimeTieredStore Tests", Label("pkg.ddc Expect(podSpec.Containers[0].Resources.Requests).To(BeNil()) Expect(podSpec.Containers[0].Resources.Limits).To(BeNil()) }) + + It("should return error when multiple processMemory levels are specified", func() { + tieredStore := &datav1alpha1.RuntimeTieredStore{ + Levels: []datav1alpha1.RuntimeTieredStoreLevel{ + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("4Gi"), + }, + }, + { + ProcessMemory: &datav1alpha1.ProcessMemoryMediumSource{ + Quota: resource.MustParse("2Gi"), + }, + }, + }, + } + + err := engine.TransformRuntimeTieredStore(tieredStore, podSpec) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("only one ProcessMemoryMediumSource")) + }) }) Context("when using HostPath medium", func() {