You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2024/06/14 06:24:20 UTC
(incubator-uniffle) 01/01: support mounting PVC for shuffleserver
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch support-pvc
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 3ec2d0a4f52e0811596e79fdc9428e5e6d2cf5fa
Author: alexcqtan <al...@tencent.com>
AuthorDate: Tue Jan 10 10:52:44 2023 +0800
support mounting PVC for shuffleserver
---
.../uniffle/v1alpha1/remoteshuffleservice_types.go | 23 +++
.../api/uniffle/v1alpha1/zz_generated.deepcopy.go | 28 +++
.../uniffle.apache.org_remoteshuffleservices.yaml | 192 +++++++++++++++++++++
.../controller/sync/shuffleserver/shuffleserver.go | 16 ++
4 files changed, 259 insertions(+)
diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
index cd53c2a12..978695343 100644
--- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
+++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
@@ -142,6 +142,29 @@ type ShuffleServerConfig struct {
// UpgradeStrategy defines upgrade strategy of shuffle servers.
UpgradeStrategy *ShuffleServerUpgradeStrategy `json:"upgradeStrategy"`
+
+ // volumeClaimTemplates is a list of claims that pods are allowed to reference.
+ // The StatefulSet controller is responsible for mapping network identities to
+ // claims in a way that maintains the identity of a pod. Every claim in
+ // this list must have at least one matching (by name) volumeMount in one
+ // container in the template. A claim in this list takes precedence over
+ // any volumes in the template, with the same name.
+ // +optional
+ VolumeClaimTemplates []ShuffleServerPersistentVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" protobuf:"bytes,4,rep,name=volumeClaimTemplates"`
+}
+
+type ShuffleServerPersistentVolumeClaimTemplate struct {
+ // May contain labels and annotations that will be copied into the PVC
+ // when creating it. No other fields are allowed and will be rejected during
+ // validation.
+ //
+ VolumeNameTemplate *string `json:"volumeNameTemplate"`
+
+ // The specification for the PersistentVolumeClaim. The entire content is
+ // copied unchanged into the PVC that gets created from this
+ // template. The same fields as in a PersistentVolumeClaim
+ // are also valid here.
+ Spec corev1.PersistentVolumeClaimSpec `json:"spec" protobuf:"bytes,2,name=spec"`
}
// ShuffleServerUpgradeStrategy defines upgrade strategy of shuffle servers.
diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
index 8d4d40def..0c7389f29 100644
--- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
+++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
@@ -443,6 +443,13 @@ func (in *ShuffleServerConfig) DeepCopyInto(out *ShuffleServerConfig) {
*out = new(ShuffleServerUpgradeStrategy)
(*in).DeepCopyInto(*out)
}
+ if in.VolumeClaimTemplates != nil {
+ in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates
+ *out = make([]ShuffleServerPersistentVolumeClaimTemplate, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerConfig.
@@ -455,6 +462,27 @@ func (in *ShuffleServerConfig) DeepCopy() *ShuffleServerConfig {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopyInto(out *ShuffleServerPersistentVolumeClaimTemplate) {
+ *out = *in
+ if in.VolumeNameTemplate != nil {
+ in, out := &in.VolumeNameTemplate, &out.VolumeNameTemplate
+ *out = new(string)
+ **out = **in
+ }
+ in.Spec.DeepCopyInto(&out.Spec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerPersistentVolumeClaimTemplate.
+func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopy() *ShuffleServerPersistentVolumeClaimTemplate {
+ if in == nil {
+ return nil
+ }
+ out := new(ShuffleServerPersistentVolumeClaimTemplate)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShuffleServerUpgradeStrategy) DeepCopyInto(out *ShuffleServerUpgradeStrategy) {
*out = *in
diff --git a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
index 314304cd5..818df4bc8 100644
--- a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
+++ b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
@@ -8362,6 +8362,198 @@ spec:
required:
- type
type: object
+ volumeClaimTemplates:
+ description: volumeClaimTemplates is a list of claims that pods
+ are allowed to reference. The StatefulSet controller is responsible
+ for mapping network identities to claims in a way that maintains
+ the identity of a pod. Every claim in this list must have at
+ least one matching (by name) volumeMount in one container in
+ the template. A claim in this list takes precedence over any
+ volumes in the template, with the same name.
+ items:
+ properties:
+ spec:
+ description: The specification for the PersistentVolumeClaim.
+ The entire content is copied unchanged into the PVC that
+ gets created from this template. The same fields as in
+ a PersistentVolumeClaim are also valid here.
+ properties:
+ accessModes:
+ description: 'AccessModes contains the desired access
+ modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'
+ items:
+ type: string
+ type: array
+ dataSource:
+ description: 'This field can be used to specify either:
+ * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot)
+ * An existing PVC (PersistentVolumeClaim) If the provisioner
+ or an external controller can support the specified
+ data source, it will create a new volume based on
+ the contents of the specified data source. If the
+ AnyVolumeDataSource feature gate is enabled, this
+ field will always have the same contents as the DataSourceRef
+ field.'
+ properties:
+ apiGroup:
+ description: APIGroup is the group for the resource
+ being referenced. If APIGroup is not specified,
+ the specified Kind must be in the core API group.
+ For any other third-party types, APIGroup is required.
+ type: string
+ kind:
+ description: Kind is the type of resource being
+ referenced
+ type: string
+ name:
+ description: Name is the name of resource being
+ referenced
+ type: string
+ required:
+ - kind
+ - name
+ type: object
+ dataSourceRef:
+ description: 'Specifies the object from which to populate
+ the volume with data, if a non-empty volume is desired.
+ This may be any local object from a non-empty API
+ group (non core object) or a PersistentVolumeClaim
+ object. When this field is specified, volume binding
+ will only succeed if the type of the specified object
+ matches some installed volume populator or dynamic
+ provisioner. This field will replace the functionality
+ of the DataSource field and as such if both fields
+ are non-empty, they must have the same value. For
+ backwards compatibility, both fields (DataSource and
+ DataSourceRef) will be set to the same value automatically
+ if one of them is empty and the other is non-empty.
+ There are two important differences between DataSource
+ and DataSourceRef: * While DataSource only allows
+ two specific types of objects, DataSourceRef allows
+ any non-core object, as well as PersistentVolumeClaim
+ objects. * While DataSource ignores disallowed values
+ (dropping them), DataSourceRef preserves all values,
+ and generates an error if a disallowed value is specified.
+ (Alpha) Using this field requires the AnyVolumeDataSource
+ feature gate to be enabled.'
+ properties:
+ apiGroup:
+ description: APIGroup is the group for the resource
+ being referenced. If APIGroup is not specified,
+ the specified Kind must be in the core API group.
+ For any other third-party types, APIGroup is required.
+ type: string
+ kind:
+ description: Kind is the type of resource being
+ referenced
+ type: string
+ name:
+ description: Name is the name of resource being
+ referenced
+ type: string
+ required:
+ - kind
+ - name
+ type: object
+ resources:
+ description: 'Resources represents the minimum resources
+ the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
+ properties:
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Limits describes the maximum amount
+ of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Requests describes the minimum amount
+ of compute resources required. If Requests is
+ omitted for a container, it defaults to Limits
+ if that is explicitly specified, otherwise to
+ an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ type: object
+ selector:
+ description: A label query over volumes to consider
+ for binding.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label
+ selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a
+ selector that contains values, a key, and an
+ operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the
+ selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship
+ to a set of values. Valid operators are
+ In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string
+ values. If the operator is In or NotIn,
+ the values array must be non-empty. If the
+ operator is Exists or DoesNotExist, the
+ values array must be empty. This array is
+ replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value}
+ pairs. A single {key,value} in the matchLabels
+ map is equivalent to an element of matchExpressions,
+ whose key field is "key", the operator is "In",
+ and the values array contains only "value". The
+ requirements are ANDed.
+ type: object
+ type: object
+ storageClassName:
+ description: 'Name of the StorageClass required by the
+ claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'
+ type: string
+ volumeMode:
+ description: volumeMode defines what type of volume
+ is required by the claim. Value of Filesystem is implied
+ when not included in claim spec.
+ type: string
+ volumeName:
+ description: VolumeName is the binding reference to
+ the PersistentVolume backing this claim.
+ type: string
+ type: object
+ volumeNameTemplate:
+ description: May contain labels and annotations that will
+ be copied into the PVC when creating it. No other fields
+ are allowed and will be rejected during validation.
+ type: string
+ required:
+ - spec
+ - volumeNameTemplate
+ type: object
+ type: array
volumeMounts:
description: VolumeMounts indicates describes mountings of volumes
within shuffle servers' container.
diff --git a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
index a96c0eba0..9d5dc17d8 100644
--- a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
+++ b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
@@ -198,6 +198,17 @@ func GenerateSts(kubeClient kubernetes.Interface, rss *unifflev1alpha1.RemoteShu
sts.Spec.Template.Spec.RuntimeClassName = rss.Spec.ShuffleServer.RuntimeClassName
}
+ // add VolumeClaimTemplates, support cloud storage
+ sts.Spec.VolumeClaimTemplates = make([]corev1.PersistentVolumeClaim, 0, len(rss.Spec.ShuffleServer.VolumeClaimTemplates))
+ for _, pvcTemplate := range rss.Spec.ShuffleServer.VolumeClaimTemplates {
+ sts.Spec.VolumeClaimTemplates = append(sts.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: *pvcTemplate.VolumeNameTemplate,
+ },
+ Spec: pvcTemplate.Spec,
+ })
+ }
+
// add init containers, the main container and other containers.
sts.Spec.Template.Spec.InitContainers = util.GenerateInitContainers(rss.Spec.ShuffleServer.RSSPodSpec)
containers := []corev1.Container{*generateMainContainer(rss)}
@@ -238,6 +249,11 @@ func generateStorageBasePath(rss *unifflev1alpha1.RemoteShuffleService) string {
}
paths = append(paths, strings.TrimSuffix(v, "/")+"/"+controllerconstants.RssDataDir)
}
+
+ for _, vm := range rss.Spec.ShuffleServer.VolumeMounts {
+ paths = append(paths, strings.TrimSuffix(vm.MountPath, "/"+"rssdata"))
+ }
+
sort.Strings(paths)
return strings.Join(paths, ",")
}