You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by sa...@apache.org on 2022/05/02 17:28:51 UTC
[incubator-heron] 01/02: [K8s] created Volume Mount in Volume Factory.
This is an automated email from the ASF dual-hosted git repository.
saadurrahman pushed a commit to branch saadurrahman/3821-Remove-Deprecated-Volumes-K8s-dev
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 7bc2f02d70374d22378ead793f94aac6efd2d77b
Author: Saad Ur Rahman <sa...@apache.org>
AuthorDate: Mon May 2 13:23:46 2022 -0400
[K8s] created Volume Mount in Volume Factory.
Generating all volume mounts from within Volume Factory for maintainability.
---
.../heron/scheduler/kubernetes/V1Controller.java | 6 +-
.../apache/heron/scheduler/kubernetes/Volumes.java | 48 ++++++---
.../heron/scheduler/kubernetes/VolumesTests.java | 110 ++++++++++++++++++++-
3 files changed, 144 insertions(+), 20 deletions(-)
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
index 338e50bda51..a065790df5d 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
@@ -1239,7 +1239,7 @@ public class V1Controller extends KubernetesController {
: mapOfOpts.entrySet()) {
final String volumeName = configs.getKey();
final V1Volume volume = Volumes.get()
- .create(Volumes.VolumeType.EmptyDir, volumeName, configs.getValue());
+ .createVolume(Volumes.VolumeType.EmptyDir, volumeName, configs.getValue());
volumes.add(volume);
volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue()));
}
@@ -1260,7 +1260,7 @@ public class V1Controller extends KubernetesController {
: mapOfOpts.entrySet()) {
final String volumeName = configs.getKey();
final V1Volume volume = Volumes.get()
- .create(Volumes.VolumeType.HostPath, volumeName, configs.getValue());
+ .createVolume(Volumes.VolumeType.HostPath, volumeName, configs.getValue());
volumes.add(volume);
volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue()));
}
@@ -1281,7 +1281,7 @@ public class V1Controller extends KubernetesController {
: mapOfOpts.entrySet()) {
final String volumeName = configs.getKey();
final V1Volume volume = Volumes.get()
- .create(Volumes.VolumeType.NetworkFileSystem, volumeName, configs.getValue());
+ .createVolume(Volumes.VolumeType.NetworkFileSystem, volumeName, configs.getValue());
volumes.add(volume);
volumeMounts.add(createVolumeMountsCLI(volumeName, configs.getValue()));
}
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
index 038dd08a7d6..39e9a658d5d 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/Volumes.java
@@ -27,6 +27,7 @@ import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeBuilder;
import io.kubernetes.client.openapi.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1VolumeMountBuilder;
final class Volumes {
@@ -34,8 +35,7 @@ final class Volumes {
EmptyDir,
HostPath,
NetworkFileSystem,
- PersistentVolumeClaim,
- VolumeMount
+ PersistentVolumeClaim
}
private final Map<VolumeType, IVolumeFactory> volumes = new HashMap<>();
@@ -57,26 +57,45 @@ final class Volumes {
* @param configs A map of configurations.
* @return Fully configured <code>V1Volume</code>.
*/
- V1Volume create(VolumeType volumeType, String volumeName,
- Map<KubernetesConstants.VolumeConfigKeys, String> configs) {
+ V1Volume createVolume(VolumeType volumeType, String volumeName,
+ Map<KubernetesConstants.VolumeConfigKeys, String> configs) {
if (volumes.containsKey(volumeType)) {
return volumes.get(volumeType).create(volumeName, configs);
}
return null;
}
- interface IVolumeFactory {
- V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs);
- }
-
- interface IVolumeMountFactory {
- V1VolumeMount create(String volumeName,
- Map<KubernetesConstants.VolumeConfigKeys, String> configs);
+ /**
+ * Generates a <code>Volume Mount</code> from specifications.
+ * @param volumeName Name of the <code>Volume</code>.
+ * @param configs Mapping of <code>Volume</code> option <code>key-value</code> configuration pairs.
+ * @return A configured <code>V1VolumeMount</code>.
+ */
+ V1VolumeMount createMount(String volumeName,
+ Map<KubernetesConstants.VolumeConfigKeys, String> configs) {
+ final V1VolumeMount volumeMount = new V1VolumeMountBuilder()
+ .withName(volumeName)
+ .build();
+ for (Map.Entry<KubernetesConstants.VolumeConfigKeys, String> config : configs.entrySet()) {
+ switch (config.getKey()) {
+ case path:
+ volumeMount.mountPath(config.getValue());
+ break;
+ case subPath:
+ volumeMount.subPath(config.getValue());
+ break;
+ case readOnly:
+ volumeMount.readOnly(Boolean.parseBoolean(config.getValue()));
+ break;
+ default:
+ break;
+ }
+ }
+ return volumeMount;
}
- interface IPersistentVolumeClaimFactory {
- V1PersistentVolumeClaim create(String volumeName,
- Map<KubernetesConstants.VolumeConfigKeys, String> configs);
+ interface IVolumeFactory {
+ V1Volume create(String volumeName, Map<KubernetesConstants.VolumeConfigKeys, String> configs);
}
static class EmptyDirVolumeFactory implements IVolumeFactory {
@@ -182,4 +201,5 @@ final class Volumes {
return volume;
}
}
+
}
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
index 10bb4739bdf..a4b8c72be02 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java
@@ -19,6 +19,8 @@
package org.apache.heron.scheduler.kubernetes;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
@@ -26,8 +28,12 @@ import com.google.common.collect.ImmutableMap;
import org.junit.Assert;
import org.junit.Test;
+import org.apache.heron.common.basics.Pair;
+
import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeBuilder;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1VolumeMountBuilder;
public class VolumesTests {
@@ -54,7 +60,7 @@ public class VolumesTests {
.build();
final V1Volume actualVolume = Volumes.get()
- .create(Volumes.VolumeType.EmptyDir, volumeName, config);
+ .createVolume(Volumes.VolumeType.EmptyDir, volumeName, config);
Assert.assertEquals("Volume Factory Empty Directory", expectedVolume, actualVolume);
}
@@ -82,7 +88,7 @@ public class VolumesTests {
.build();
final V1Volume actualVolume = Volumes.get()
- .create(Volumes.VolumeType.HostPath, volumeName, config);
+ .createVolume(Volumes.VolumeType.HostPath, volumeName, config);
Assert.assertEquals("Volume Factory Host Path", expectedVolume, actualVolume);
}
@@ -113,8 +119,106 @@ public class VolumesTests {
.build();
final V1Volume actualVolume = Volumes.get()
- .create(Volumes.VolumeType.NetworkFileSystem, volumeName, config);
+ .createVolume(Volumes.VolumeType.NetworkFileSystem, volumeName, config);
Assert.assertEquals("Volume Factory Network File System", expectedVolume, actualVolume);
}
+
+ @Test
+ public void testVolumeMount() {
+ final String volumeNamePVC = "volume-name-pvc";
+ final String volumeNameHostPath = "volume-name-host-path";
+ final String volumeNameEmptyDir = "volume-name-empty-dir";
+ final String volumeNameNFS = "volume-name-nfs";
+ final String value = "inserted-value";
+
+ // Test case container.
+ // Input: [0] volume name, [1] volume options
+ // Output: The expected <V1VolumeMount>.
+ final List<KubernetesUtils.TestTuple<
+ Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>, V1VolumeMount>> testCases =
+ new LinkedList<>();
+
+ // PVC.
+ final Map<KubernetesConstants.VolumeConfigKeys, String> configPVC =
+ ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder()
+ .put(KubernetesConstants.VolumeConfigKeys.claimName, value)
+ .put(KubernetesConstants.VolumeConfigKeys.storageClassName, value)
+ .put(KubernetesConstants.VolumeConfigKeys.sizeLimit, value)
+ .put(KubernetesConstants.VolumeConfigKeys.accessModes, value)
+ .put(KubernetesConstants.VolumeConfigKeys.volumeMode, value)
+ .put(KubernetesConstants.VolumeConfigKeys.path, value)
+ .put(KubernetesConstants.VolumeConfigKeys.subPath, value)
+ .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true")
+ .build();
+ final V1VolumeMount volumeMountPVC = new V1VolumeMountBuilder()
+ .withName(volumeNamePVC)
+ .withMountPath(value)
+ .withSubPath(value)
+ .withReadOnly(true)
+ .build();
+ testCases.add(new KubernetesUtils.TestTuple<>("PVC volume mount",
+ new Pair<>(volumeNamePVC, configPVC), volumeMountPVC));
+
+ // Host Path.
+ final Map<KubernetesConstants.VolumeConfigKeys, String> configHostPath =
+ ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder()
+ .put(KubernetesConstants.VolumeConfigKeys.type, "DirectoryOrCreate")
+ .put(KubernetesConstants.VolumeConfigKeys.pathOnHost, value)
+ .put(KubernetesConstants.VolumeConfigKeys.path, value)
+ .put(KubernetesConstants.VolumeConfigKeys.subPath, value)
+ .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true")
+ .build();
+ final V1VolumeMount volumeMountHostPath = new V1VolumeMountBuilder()
+ .withName(volumeNameHostPath)
+ .withMountPath(value)
+ .withSubPath(value)
+ .withReadOnly(true)
+ .build();
+ testCases.add(new KubernetesUtils.TestTuple<>("Host Path volume mount",
+ new Pair<>(volumeNameHostPath, configHostPath), volumeMountHostPath));
+
+ // Empty Dir.
+ final Map<KubernetesConstants.VolumeConfigKeys, String> configEmptyDir =
+ ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder()
+ .put(KubernetesConstants.VolumeConfigKeys.sizeLimit, value)
+ .put(KubernetesConstants.VolumeConfigKeys.medium, "Memory")
+ .put(KubernetesConstants.VolumeConfigKeys.path, value)
+ .put(KubernetesConstants.VolumeConfigKeys.subPath, value)
+ .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true")
+ .build();
+ final V1VolumeMount volumeMountEmptyDir = new V1VolumeMountBuilder()
+ .withName(volumeNameEmptyDir)
+ .withMountPath(value)
+ .withSubPath(value)
+ .withReadOnly(true)
+ .build();
+ testCases.add(new KubernetesUtils.TestTuple<>("Empty Dir volume mount",
+ new Pair<>(volumeNameEmptyDir, configEmptyDir), volumeMountEmptyDir));
+
+ // NFS.
+ final Map<KubernetesConstants.VolumeConfigKeys, String> configNFS =
+ ImmutableMap.<KubernetesConstants.VolumeConfigKeys, String>builder()
+ .put(KubernetesConstants.VolumeConfigKeys.server, "nfs.server.address")
+ .put(KubernetesConstants.VolumeConfigKeys.readOnly, "true")
+ .put(KubernetesConstants.VolumeConfigKeys.pathOnNFS, value)
+ .put(KubernetesConstants.VolumeConfigKeys.path, value)
+ .put(KubernetesConstants.VolumeConfigKeys.subPath, value)
+ .build();
+ final V1VolumeMount volumeMountNFS = new V1VolumeMountBuilder()
+ .withName(volumeNameNFS)
+ .withMountPath(value)
+ .withSubPath(value)
+ .withReadOnly(true)
+ .build();
+ testCases.add(new KubernetesUtils.TestTuple<>("NFS volume mount",
+ new Pair<>(volumeNameNFS, configNFS), volumeMountNFS));
+
+ // Test loop.
+ for (KubernetesUtils.TestTuple<Pair<String, Map<KubernetesConstants.VolumeConfigKeys, String>>,
+ V1VolumeMount> testCase : testCases) {
+ V1VolumeMount actual = Volumes.get().createMount(testCase.input.first, testCase.input.second);
+ Assert.assertEquals(testCase.description, testCase.expected, actual);
+ }
+ }
}