You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2024/06/14 06:24:19 UTC
(incubator-uniffle) branch support-pvc created (now 3ec2d0a4f)
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a change to branch support-pvc
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
at 3ec2d0a4f support mounting PVC for shuffleserver
This branch includes the following new commits:
new 3ec2d0a4f support mounting PVC for shuffleserver
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-uniffle) 01/01: support mounting PVC for shuffleserver
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch support-pvc
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 3ec2d0a4f52e0811596e79fdc9428e5e6d2cf5fa
Author: alexcqtan <al...@tencent.com>
AuthorDate: Tue Jan 10 10:52:44 2023 +0800
support mounting PVC for shuffleserver
---
.../uniffle/v1alpha1/remoteshuffleservice_types.go | 23 +++
.../api/uniffle/v1alpha1/zz_generated.deepcopy.go | 28 +++
.../uniffle.apache.org_remoteshuffleservices.yaml | 192 +++++++++++++++++++++
.../controller/sync/shuffleserver/shuffleserver.go | 16 ++
4 files changed, 259 insertions(+)
diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
index cd53c2a12..978695343 100644
--- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
+++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go
@@ -142,6 +142,29 @@ type ShuffleServerConfig struct {
// UpgradeStrategy defines upgrade strategy of shuffle servers.
UpgradeStrategy *ShuffleServerUpgradeStrategy `json:"upgradeStrategy"`
+
+ // volumeClaimTemplates is a list of claims that pods are allowed to reference.
+ // The StatefulSet controller is responsible for mapping network identities to
+ // claims in a way that maintains the identity of a pod. Every claim in
+ // this list must have at least one matching (by name) volumeMount in one
+ // container in the template. A claim in this list takes precedence over
+ // any volumes in the template, with the same name.
+ // +optional
+ VolumeClaimTemplates []ShuffleServerPersistentVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" protobuf:"bytes,4,rep,name=volumeClaimTemplates"`
+}
+
+type ShuffleServerPersistentVolumeClaimTemplate struct {
+ // May contain labels and annotations that will be copied into the PVC
+ // when creating it. No other fields are allowed and will be rejected during
+ // validation.
+ //
+ VolumeNameTemplate *string `json:"volumeNameTemplate"`
+
+ // The specification for the PersistentVolumeClaim. The entire content is
+ // copied unchanged into the PVC that gets created from this
+ // template. The same fields as in a PersistentVolumeClaim
+ // are also valid here.
+ Spec corev1.PersistentVolumeClaimSpec `json:"spec" protobuf:"bytes,2,name=spec"`
}
// ShuffleServerUpgradeStrategy defines upgrade strategy of shuffle servers.
diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
index 8d4d40def..0c7389f29 100644
--- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
+++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go
@@ -443,6 +443,13 @@ func (in *ShuffleServerConfig) DeepCopyInto(out *ShuffleServerConfig) {
*out = new(ShuffleServerUpgradeStrategy)
(*in).DeepCopyInto(*out)
}
+ if in.VolumeClaimTemplates != nil {
+ in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates
+ *out = make([]ShuffleServerPersistentVolumeClaimTemplate, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerConfig.
@@ -455,6 +462,27 @@ func (in *ShuffleServerConfig) DeepCopy() *ShuffleServerConfig {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopyInto(out *ShuffleServerPersistentVolumeClaimTemplate) {
+ *out = *in
+ if in.VolumeNameTemplate != nil {
+ in, out := &in.VolumeNameTemplate, &out.VolumeNameTemplate
+ *out = new(string)
+ **out = **in
+ }
+ in.Spec.DeepCopyInto(&out.Spec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerPersistentVolumeClaimTemplate.
+func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopy() *ShuffleServerPersistentVolumeClaimTemplate {
+ if in == nil {
+ return nil
+ }
+ out := new(ShuffleServerPersistentVolumeClaimTemplate)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShuffleServerUpgradeStrategy) DeepCopyInto(out *ShuffleServerUpgradeStrategy) {
*out = *in
diff --git a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
index 314304cd5..818df4bc8 100644
--- a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
+++ b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml
@@ -8362,6 +8362,198 @@ spec:
required:
- type
type: object
+ volumeClaimTemplates:
+ description: volumeClaimTemplates is a list of claims that pods
+ are allowed to reference. The StatefulSet controller is responsible
+ for mapping network identities to claims in a way that maintains
+ the identity of a pod. Every claim in this list must have at
+ least one matching (by name) volumeMount in one container in
+ the template. A claim in this list takes precedence over any
+ volumes in the template, with the same name.
+ items:
+ properties:
+ spec:
+ description: The specification for the PersistentVolumeClaim.
+ The entire content is copied unchanged into the PVC that
+ gets created from this template. The same fields as in
+ a PersistentVolumeClaim are also valid here.
+ properties:
+ accessModes:
+ description: 'AccessModes contains the desired access
+ modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'
+ items:
+ type: string
+ type: array
+ dataSource:
+ description: 'This field can be used to specify either:
+ * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot)
+ * An existing PVC (PersistentVolumeClaim) If the provisioner
+ or an external controller can support the specified
+ data source, it will create a new volume based on
+ the contents of the specified data source. If the
+ AnyVolumeDataSource feature gate is enabled, this
+ field will always have the same contents as the DataSourceRef
+ field.'
+ properties:
+ apiGroup:
+ description: APIGroup is the group for the resource
+ being referenced. If APIGroup is not specified,
+ the specified Kind must be in the core API group.
+ For any other third-party types, APIGroup is required.
+ type: string
+ kind:
+ description: Kind is the type of resource being
+ referenced
+ type: string
+ name:
+ description: Name is the name of resource being
+ referenced
+ type: string
+ required:
+ - kind
+ - name
+ type: object
+ dataSourceRef:
+ description: 'Specifies the object from which to populate
+ the volume with data, if a non-empty volume is desired.
+ This may be any local object from a non-empty API
+ group (non core object) or a PersistentVolumeClaim
+ object. When this field is specified, volume binding
+ will only succeed if the type of the specified object
+ matches some installed volume populator or dynamic
+ provisioner. This field will replace the functionality
+ of the DataSource field and as such if both fields
+ are non-empty, they must have the same value. For
+ backwards compatibility, both fields (DataSource and
+ DataSourceRef) will be set to the same value automatically
+ if one of them is empty and the other is non-empty.
+ There are two important differences between DataSource
+ and DataSourceRef: * While DataSource only allows
+ two specific types of objects, DataSourceRef allows
+ any non-core object, as well as PersistentVolumeClaim
+ objects. * While DataSource ignores disallowed values
+ (dropping them), DataSourceRef preserves all values,
+ and generates an error if a disallowed value is specified.
+ (Alpha) Using this field requires the AnyVolumeDataSource
+ feature gate to be enabled.'
+ properties:
+ apiGroup:
+ description: APIGroup is the group for the resource
+ being referenced. If APIGroup is not specified,
+ the specified Kind must be in the core API group.
+ For any other third-party types, APIGroup is required.
+ type: string
+ kind:
+ description: Kind is the type of resource being
+ referenced
+ type: string
+ name:
+ description: Name is the name of resource being
+ referenced
+ type: string
+ required:
+ - kind
+ - name
+ type: object
+ resources:
+ description: 'Resources represents the minimum resources
+ the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
+ properties:
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Limits describes the maximum amount
+ of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Requests describes the minimum amount
+ of compute resources required. If Requests is
+ omitted for a container, it defaults to Limits
+ if that is explicitly specified, otherwise to
+ an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ type: object
+ selector:
+ description: A label query over volumes to consider
+ for binding.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label
+ selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a
+ selector that contains values, a key, and an
+ operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the
+ selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship
+ to a set of values. Valid operators are
+ In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string
+ values. If the operator is In or NotIn,
+ the values array must be non-empty. If the
+ operator is Exists or DoesNotExist, the
+ values array must be empty. This array is
+ replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value}
+ pairs. A single {key,value} in the matchLabels
+ map is equivalent to an element of matchExpressions,
+ whose key field is "key", the operator is "In",
+ and the values array contains only "value". The
+ requirements are ANDed.
+ type: object
+ type: object
+ storageClassName:
+ description: 'Name of the StorageClass required by the
+ claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'
+ type: string
+ volumeMode:
+ description: volumeMode defines what type of volume
+ is required by the claim. Value of Filesystem is implied
+ when not included in claim spec.
+ type: string
+ volumeName:
+ description: VolumeName is the binding reference to
+ the PersistentVolume backing this claim.
+ type: string
+ type: object
+ volumeNameTemplate:
+ description: May contain labels and annotations that will
+ be copied into the PVC when creating it. No other fields
+ are allowed and will be rejected during validation.
+ type: string
+ required:
+ - spec
+ - volumeNameTemplate
+ type: object
+ type: array
volumeMounts:
description: VolumeMounts indicates describes mountings of volumes
within shuffle servers' container.
diff --git a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
index a96c0eba0..9d5dc17d8 100644
--- a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
+++ b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go
@@ -198,6 +198,17 @@ func GenerateSts(kubeClient kubernetes.Interface, rss *unifflev1alpha1.RemoteShu
sts.Spec.Template.Spec.RuntimeClassName = rss.Spec.ShuffleServer.RuntimeClassName
}
+ // add VolumeClaimTemplates, support cloud storage
+ sts.Spec.VolumeClaimTemplates = make([]corev1.PersistentVolumeClaim, 0, len(rss.Spec.ShuffleServer.VolumeClaimTemplates))
+ for _, pvcTemplate := range rss.Spec.ShuffleServer.VolumeClaimTemplates {
+ sts.Spec.VolumeClaimTemplates = append(sts.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: *pvcTemplate.VolumeNameTemplate,
+ },
+ Spec: pvcTemplate.Spec,
+ })
+ }
+
// add init containers, the main container and other containers.
sts.Spec.Template.Spec.InitContainers = util.GenerateInitContainers(rss.Spec.ShuffleServer.RSSPodSpec)
containers := []corev1.Container{*generateMainContainer(rss)}
@@ -238,6 +249,11 @@ func generateStorageBasePath(rss *unifflev1alpha1.RemoteShuffleService) string {
}
paths = append(paths, strings.TrimSuffix(v, "/")+"/"+controllerconstants.RssDataDir)
}
+
+ for _, vm := range rss.Spec.ShuffleServer.VolumeMounts {
+ paths = append(paths, strings.TrimSuffix(vm.MountPath, "/"+"rssdata"))
+ }
+
sort.Strings(paths)
return strings.Join(paths, ",")
}