You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2021/01/27 04:44:35 UTC
[submarine] branch master updated: SUBMARINE-685. Retain the data
in notebook pod even if notebook server is destroyed
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 9a2110a SUBMARINE-685. Retain the data in notebook pod even if notebook server is destroyed
9a2110a is described below
commit 9a2110aaf70b010454413f67744903d54a966ffc
Author: Charles <ch...@gmail.com>
AuthorDate: Thu Jan 14 15:27:42 2021 +0800
SUBMARINE-685. Retain the data in notebook pod even if notebook server is destroyed
### What is this PR for?
Add 10GB persistent volume for each notebook which mounting path is /home/jovyan/workspace.
I use local persistent volume which mounting path of node is /mnt since Submarine do not use any storage class yet.
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
[SUBMARINE-685](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-685?filter=allopenissues)
### How should this be tested?
[Travis-CI](https://travis-ci.com/github/charlie0220/submarine/builds/210378125)
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Charles <ch...@gmail.com>
Signed-off-by: Liu Xun <li...@apache.org>
Closes #481 from charlie0220/SUBMARINE-685 and squashes the following commits:
c4821a5 [Charles] add comments for Thread.sleep and some variables name
0884d43 [Charles] add Thread.sleep in NotebookRestApiIT.java waiting for deleting pvc and pv
1f842bf [Charles] SUBMARINE-685. Retain the data in notebook pod even if notebook server is destroyed.
---
.../server/submitter/k8s/K8sSubmitter.java | 59 ++++++++++++++--------
.../submitter/k8s/parser/NotebookSpecParser.java | 20 ++++++++
.../server/submitter/k8s/util/NotebookUtils.java | 4 ++
.../apache/submarine/rest/NotebookRestApiIT.java | 5 ++
4 files changed, 68 insertions(+), 20 deletions(-)
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 5521f99..6d5b50d 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -43,6 +43,7 @@ import io.kubernetes.client.models.V1PersistentVolumeClaim;
import io.kubernetes.client.models.V1Pod;
import io.kubernetes.client.models.V1PodList;
import io.kubernetes.client.models.V1Service;
+import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
import io.kubernetes.client.models.V1Status;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
@@ -129,12 +130,17 @@ public class K8sSubmitter implements Submitter {
public Experiment createExperiment(ExperimentSpec spec) throws SubmarineRuntimeException {
Experiment experiment;
final String name = spec.getMeta().getName();
+ final String pvName = TensorboardUtils.PV_PREFIX + name;
+ final String hostPath = TensorboardUtils.HOST_PREFIX + name;
+ final String storage = TensorboardUtils.STORAGE;
+ final String pvcName = TensorboardUtils.PVC_PREFIX + name;
+ final String volume = TensorboardUtils.PV_PREFIX + name;
try {
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
- createTFBoardPersistentVolume(name);
- createTFBoardPersistentVolumeClaim(name, spec.getMeta().getNamespace());
+ createPersistentVolume(pvName, hostPath, storage);
+ createPersistentVolumeClaim(pvcName, spec.getMeta().getNamespace(), volume, storage);
createTFBoard(name, spec.getMeta().getNamespace());
Object object = api.createNamespacedCustomObject(mlJob.getGroup(), mlJob.getVersion(),
@@ -188,12 +194,14 @@ public class K8sSubmitter implements Submitter {
public Experiment deleteExperiment(ExperimentSpec spec) throws SubmarineRuntimeException {
Experiment experiment;
final String name = spec.getMeta().getName(); // spec.getMeta().getEnvVars().get(RestConstants.JOB_ID);
+ final String pvName = TensorboardUtils.PV_PREFIX + name;
+ final String pvcName = TensorboardUtils.PVC_PREFIX + name;
try {
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
- deleteTFBoardPersistentVolume(name);
- deleteTFBoardPersistentVolumeClaim(name, spec.getMeta().getNamespace());
+ deletePersistentVolume(pvName);
+ deletePersistentVolumeClaim(pvcName, spec.getMeta().getNamespace());
deleteTFBoard(name, spec.getMeta().getNamespace());
Object object = api.deleteNamespacedCustomObject(mlJob.getGroup(), mlJob.getVersion(),
@@ -277,12 +285,29 @@ public class K8sSubmitter implements Submitter {
@Override
public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
Notebook notebook;
+ final String name = spec.getMeta().getName();
+ final String pvName = NotebookUtils.PV_PREFIX + name;
+ final String host = NotebookUtils.HOST_PATH;
+ final String storage = NotebookUtils.STORAGE;
+ final String pvcName = NotebookUtils.PVC_PREFIX + name;
try {
// create notebook custom resource
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
Map<String, String> labels = new HashMap<>();
labels.put(NotebookCR.NOTEBOOK_OWNER_SELECTOR_KET, spec.getMeta().getOwnerId());
notebookCR.getMetadata().setLabels(labels);
+
+ // create persistent volume
+ createPersistentVolume(pvName, host, storage);
+
+ // create persistent volume claim
+ createPersistentVolumeClaim(pvcName, spec.getMeta().getNamespace(), pvName, storage);
+
+ // bind persistent volume claim
+ V1PersistentVolumeClaimVolumeSource pvcSource = new V1PersistentVolumeClaimVolumeSource()
+ .claimName(pvcName);
+ notebookCR.getSpec().getTemplate().getSpec().getVolumes().get(0).persistentVolumeClaim(pvcSource);
+
Object object = api.createNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(), notebookCR, "true");
notebook = NotebookUtils.parseObject(object, NotebookUtils.ParseOpt.PARSE_OPT_CREATE);
@@ -319,6 +344,9 @@ public class K8sSubmitter implements Submitter {
@Override
public Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
Notebook notebook;
+ final String name = spec.getMeta().getName();
+ final String pvName = NotebookUtils.PV_PREFIX + name;
+ final String pvcName = NotebookUtils.PVC_PREFIX + name;
try {
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
Object object = api.deleteNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
@@ -328,6 +356,8 @@ public class K8sSubmitter implements Submitter {
null, null, null);
notebook = NotebookUtils.parseObject(object, NotebookUtils.ParseOpt.PARSE_OPT_DELETE);
deleteIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName());
+ deletePersistentVolumeClaim(pvcName, spec.getMeta().getNamespace());
+ deletePersistentVolume(pvName);
} catch (ApiException e) {
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
@@ -411,11 +441,7 @@ public class K8sSubmitter implements Submitter {
}
}
- public void createTFBoardPersistentVolume(String name) throws ApiException {
- final String pvName = TensorboardUtils.PV_PREFIX + name;
- final String hostPath = TensorboardUtils.HOST_PREFIX + name;
- final String storage = TensorboardUtils.STORAGE;
-
+ public void createPersistentVolume(String pvName, String hostPath, String storage) throws ApiException {
V1PersistentVolume pv = VolumeSpecParser.parsePersistentVolume(pvName, hostPath, storage);
try {
@@ -426,14 +452,12 @@ public class K8sSubmitter implements Submitter {
}
}
- public void deleteTFBoardPersistentVolume(String name) throws ApiException {
+ public void deletePersistentVolume(String pvName) throws ApiException {
/*
This version of Kubernetes-client/java has bug here.
It will trigger exception as in https://github.com/kubernetes-client/java/issues/86
but it can still work fine and delete the PV.
*/
- final String pvName = TensorboardUtils.PV_PREFIX + name;
-
try {
V1Status result = coreApi.deletePersistentVolume(
pvName, "true", null,
@@ -454,11 +478,8 @@ public class K8sSubmitter implements Submitter {
}
}
- public void createTFBoardPersistentVolumeClaim(String name, String namespace) throws ApiException {
- final String pvcName = TensorboardUtils.PVC_PREFIX + name;
- final String volume = TensorboardUtils.PV_PREFIX + name;
- final String storage = TensorboardUtils.STORAGE;
-
+ public void createPersistentVolumeClaim(String pvcName, String namespace, String volume, String storage)
+ throws ApiException {
V1PersistentVolumeClaim pvc = VolumeSpecParser.parsePersistentVolumeClaim(pvcName, volume, storage);
try {
@@ -471,14 +492,12 @@ public class K8sSubmitter implements Submitter {
}
}
- public void deleteTFBoardPersistentVolumeClaim(String name, String namespace) throws ApiException {
+ public void deletePersistentVolumeClaim(String pvcName, String namespace) throws ApiException {
/*
This version of Kubernetes-client/java has bug here.
It will trigger exception as in https://github.com/kubernetes-client/java/issues/86
but it can still work fine and delete the PVC
*/
- final String pvcName = TensorboardUtils.PVC_PREFIX + name;
-
try {
V1Status result = coreApi.deleteNamespacedPersistentVolumeClaim(
pvcName, namespace, "true",
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index e0889b4..bdb4b8b 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -26,6 +26,8 @@ import io.kubernetes.client.models.V1ObjectMeta;
import io.kubernetes.client.models.V1PodTemplateSpec;
import io.kubernetes.client.models.V1PodSpec;
import io.kubernetes.client.models.V1ResourceRequirements;
+import io.kubernetes.client.models.V1Volume;
+import io.kubernetes.client.models.V1VolumeMount;
import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
@@ -132,9 +134,27 @@ public class NotebookSpecParser {
container.setResources(resources);
}
+ // Volume spec
+ final String DEFAULT_MOUNT_PATH = "/home/jovyan/workspace";
+
+ List<V1VolumeMount> volumeMountList = new ArrayList<>();
+ V1VolumeMount volumeMount = new V1VolumeMount();
+ volumeMount.setMountPath(DEFAULT_MOUNT_PATH);
+ volumeMount.setName("notebook-pv-" + notebookSpec.getMeta().getName());
+ volumeMountList.add(volumeMount);
+ container.setVolumeMounts(volumeMountList);
+
containers.add(container);
podSpec.setContainers(containers);
+ // create volume object for persistent volume
+ List<V1Volume> volumeList = new ArrayList<>();
+ V1Volume volume = new V1Volume();
+ String volumeName = "notebook-pv-" + notebookSpec.getMeta().getName();
+ volume.setName(volumeName);
+ volumeList.add(volume);
+ podSpec.setVolumes(volumeList);
+
podTemplateSpec.setSpec(podSpec);
return podTemplateSpec;
}
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
index cf33ff7..33f83fe 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
@@ -43,6 +43,10 @@ import org.slf4j.LoggerFactory;
public class NotebookUtils {
private static final Logger LOG = LoggerFactory.getLogger(NotebookUtils.class);
+ public static final String STORAGE = "10Gi";
+ public static final String PV_PREFIX = "notebook-pv-";
+ public static final String PVC_PREFIX = "notebook-pvc-";
+ public static final String HOST_PATH = "/mnt";
public enum ParseOpt {
PARSE_OPT_CREATE,
diff --git a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
index d47adcf..e956477 100644
--- a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
+++ b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
@@ -56,6 +56,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.FileReader;
import java.io.IOException;
+import java.lang.Thread;
import java.util.Date;
@SuppressWarnings("rawtypes")
@@ -187,6 +188,8 @@ public class NotebookRestApiIT extends AbstractSubmarineServerTest {
gson.fromJson(gson.toJson(jsonResponse.getResult()), Environment.class);
Assert.assertEquals(ENV_NAME, getEnvironment.getEnvironmentSpec().getName());
+ // waiting for deleting previous persistent volume
+ Thread.sleep(15000);
// create notebook instances
LOG.info("Create notebook servers by Notebook REST API");
String body = loadContent("notebook/notebook-req.json");
@@ -223,6 +226,8 @@ public class NotebookRestApiIT extends AbstractSubmarineServerTest {
}
private void runTest(String body, String contentType) throws Exception {
+ // waiting for deleting previous persistent volume
+ Thread.sleep(15000);
// create
LOG.info("Create a notebook server by Notebook REST API");
PostMethod postMethod = httpPost(BASE_API_PATH, body, contentType);
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org