You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by sa...@apache.org on 2022/05/02 19:22:52 UTC
[incubator-heron] branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev created (now 35c023e1061)
This is an automated email from the ASF dual-hosted git repository.
saadurrahman pushed a change to branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
at 35c023e1061 [K8s] created PVC generation in Volume Factory.
This branch includes the following new commits:
new 35c023e1061 [K8s] created PVC generation in Volume Factory.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-heron] 01/01: [K8s] created PVC generation in Volume Factory.
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
saadurrahman pushed a commit to branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 35c023e106186fe972a82f6ce2c4d7bc9e35a409
Author: Saad Ur Rahman <sa...@apache.org>
AuthorDate: Mon May 2 15:22:06 2022 -0400
[K8s] created PVC generation in Volume Factory.
---
.../apache/heron/scheduler/kubernetes/Volumes.java | 52 ++++++++-
.../heron/scheduler/kubernetes/VolumesTests.java | 121 ++++++++++++++++++++-
2 files changed, 170 insertions(+), 3 deletions(-)
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
index 39e9a658d5d..d9e641f78cc 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
@@ -19,11 +19,14 @@
package org.apache.heron.scheduler.kubernetes;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder;
+import io.kubernetes.client.openapi.models.V1ResourceRequirements;
import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeBuilder;
import io.kubernetes.client.openapi.models.V1VolumeMount;
@@ -34,8 +37,7 @@ final class Volumes {
public enum VolumeType {
EmptyDir,
HostPath,
- NetworkFileSystem,
- PersistentVolumeClaim
+ NetworkFileSystem
}
private final Map<VolumeType, IVolumeFactory> volumes = new HashMap<>();
@@ -94,6 +96,52 @@ final class Volumes {
return volumeMount;
}
+ /**
+ * Generates <code>Persistent Volume Claims Templates</code> from a mapping of <code>Volumes</code>
+ * to <code>key-value</code> pairs of configuration options and values.
+ * @param claimName Name to be assigned to <code>Persistent Volume Claims Template</code>.
+ * @param labels Labels to be attached to the <code>Persistent Volume Claims Template</code>.
+ * @param configs <code>Volume</code> to configuration <code>key-value</code> mappings.
+ * @return Fully populated dynamically backed <code>Persistent Volume Claims</code>.
+ */
+ V1PersistentVolumeClaim createPersistentVolumeClaim(String claimName, Map<String, String> labels,
+ Map<KubernetesConstants.VolumeConfigKeys, String> configs) {
+ V1PersistentVolumeClaim claim = new V1PersistentVolumeClaimBuilder()
+ .withNewMetadata()
+ .withName(claimName)
+ .withLabels(labels)
+ .endMetadata()
+ .withNewSpec()
+ .withStorageClassName("")
+ .endSpec()
+ .build();
+
+ // Populate PVC options.
+ for (Map.Entry<KubernetesConstants.VolumeConfigKeys, String> option : configs.entrySet()) {
+ String optionValue = option.getValue();
+ switch(option.getKey()) {
+ case storageClassName:
+ claim.getSpec().setStorageClassName(optionValue);
+ break;
+ case sizeLimit:
+ claim.getSpec().setResources(
+ new V1ResourceRequirements()
+ .putRequestsItem("storage", new Quantity(optionValue)));
+ break;
+ case accessModes:
+ claim.getSpec().setAccessModes(Arrays.asList(optionValue.split(",")));
+ break;
+ case volumeMode:
+ claim.getSpec().setVolumeMode(optionValue);
+ break;
+ // Valid ignored options not used in a PVC.
+ default:
+ break;
+ }
+ }
+ return claim;
+ }
+
interface IVolumeFactory {
V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs);
}
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
index a4b8c72be02..1cd7889741b 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
@@ -19,6 +19,9 @@
package org.apache.heron.scheduler.kubernetes;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -30,6 +33,9 @@ import org.junit.Test;
import org.apache.heron.common.basics.Pair;
+import io.kubernetes.client.custom.Quantity;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder;
import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeBuilder;
import io.kubernetes.client.openapi.models.V1VolumeMount;
@@ -216,9 +222,122 @@ public class VolumesTests {
// Test loop.
for (KubernetesUtils.TestTuple<Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>,
- V1VolumeMount> testCase : testCases) {
+ V1VolumeMount> testCase : testCases) {
V1VolumeMount actual = Volumes.get().createMount(testCase.input.first, testCase.input.second);
Assert.assertEquals(testCase.description, testCase.expected, actual);
}
}
+
+ @Test
+ public void testPersistentVolumeClaim() {
+ final String topologyName = "topology-name";
+ final String volumeNameOne = "volume-name-one";
+ final String volumeNameTwo = "volume-name-two";
+ final String volumeNameStatic = "volume-name-static";
+ final String claimNameOne = "OnDemand";
+ final String claimNameTwo = "claim-name-two";
+ final String claimNameStatic = "OnDEmaND";
+ final String storageClassName = "storage-class-name";
+ final String sizeLimit = "555Gi";
+ final String accessModesList = "ReadWriteOnce,ReadOnlyMany,ReadWriteMany";
+ final String accessModes = "ReadOnlyMany";
+ final String volumeMode = "VolumeMode";
+ final String path = "/path/to/mount/";
+ final String subPath = "/sub/path/to/mount/";
+ final Map<String, String> labels = V1Controller.getPersistentVolumeClaimLabels(topologyName);
+
+ final Map<KubernetesConstants.VolumeConfigKeys, String> volOneConfig =
+ new HashMap<KubernetesConstants.VolumeConfigKeys, String>() {
+ {
+ put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameOne);
+ put(KubernetesConstants.VolumeConfigKeys.storageClassName, storageClassName);
+ put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit);
+ put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModesList);
+ put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode);
+ put(KubernetesConstants.VolumeConfigKeys.path, path);
+ }
+ };
+
+ final Map<KubernetesConstants.VolumeConfigKeys, String> volTwoConfig =
+ new HashMap<KubernetesConstants.VolumeConfigKeys, String>() {
+ {
+ put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameTwo);
+ put(KubernetesConstants.VolumeConfigKeys.storageClassName, storageClassName);
+ put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit);
+ put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModesList);
+ put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode);
+ put(KubernetesConstants.VolumeConfigKeys.path, path);
+ put(KubernetesConstants.VolumeConfigKeys.subPath, subPath);
+ }
+ };
+
+ final Map<KubernetesConstants.VolumeConfigKeys, String> volStaticConfig =
+ new HashMap<KubernetesConstants.VolumeConfigKeys, String>() {
+ {
+ put(KubernetesConstants.VolumeConfigKeys.claimName, claimNameStatic);
+ put(KubernetesConstants.VolumeConfigKeys.sizeLimit, sizeLimit);
+ put(KubernetesConstants.VolumeConfigKeys.accessModes, accessModes);
+ put(KubernetesConstants.VolumeConfigKeys.volumeMode, volumeMode);
+ put(KubernetesConstants.VolumeConfigKeys.path, path);
+ put(KubernetesConstants.VolumeConfigKeys.subPath, subPath);
+ }
+ };
+
+ final V1PersistentVolumeClaim claimOne = new V1PersistentVolumeClaimBuilder()
+ .withNewMetadata()
+ .withName(volumeNameOne)
+ .withLabels(labels)
+ .endMetadata()
+ .withNewSpec()
+ .withStorageClassName(storageClassName)
+ .withAccessModes(Arrays.asList(accessModesList.split(",")))
+ .withVolumeMode(volumeMode)
+ .withNewResources()
+ .addToRequests("storage", new Quantity(sizeLimit))
+ .endResources()
+ .endSpec()
+ .build();
+
+ final V1PersistentVolumeClaim claimTwo = new V1PersistentVolumeClaimBuilder()
+ .withNewMetadata()
+ .withName(volumeNameTwo)
+ .withLabels(labels)
+ .endMetadata()
+ .withNewSpec()
+ .withStorageClassName(storageClassName)
+ .withAccessModes(Arrays.asList(accessModesList.split(",")))
+ .withVolumeMode(volumeMode)
+ .withNewResources()
+ .addToRequests("storage", new Quantity(sizeLimit))
+ .endResources()
+ .endSpec()
+ .build();
+
+ final V1PersistentVolumeClaim claimStatic = new V1PersistentVolumeClaimBuilder()
+ .withNewMetadata()
+ .withName(volumeNameStatic)
+ .withLabels(labels)
+ .endMetadata()
+ .withNewSpec()
+ .withStorageClassName("")
+ .withAccessModes(Collections.singletonList(accessModes))
+ .withVolumeMode(volumeMode)
+ .withNewResources()
+ .addToRequests("storage", new Quantity(sizeLimit))
+ .endResources()
+ .endSpec()
+ .build();
+
+ final V1PersistentVolumeClaim actualPVCOne = Volumes.get()
+ .createPersistentVolumeClaim(volumeNameOne, labels, volOneConfig);
+ Assert.assertEquals("Volume one PVC", claimOne, actualPVCOne);
+
+ final V1PersistentVolumeClaim actualPVCTwo = Volumes.get()
+ .createPersistentVolumeClaim(volumeNameTwo, labels, volTwoConfig);
+ Assert.assertEquals("Volume two PVC", claimTwo, actualPVCTwo);
+
+ final V1PersistentVolumeClaim actualPVCStatic = Volumes.get()
+ .createPersistentVolumeClaim(volumeNameStatic, labels, volStaticConfig);
+ Assert.assertEquals("Volume static PVC", claimStatic, actualPVCStatic);
+ }
}