You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2020/09/22 11:43:29 UTC
[submarine] branch master updated: SUBMARINE-597. Support for SSH
based git sync mode
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 5bdec19 SUBMARINE-597. Support for SSH based git sync mode
5bdec19 is described below
commit 5bdec19e932aaa11eb191ef159858b60851340f0
Author: Manikandan R <ma...@gmail.com>
AuthorDate: Wed Sep 16 08:26:50 2020 +0530
SUBMARINE-597. Support for SSH based git sync mode
### What is this PR for?
Clone the source code from git repository using SSH based access. /code dir is the location to store the source code. SSH based access requires SSH host private key and known hosts to be passed through /etc/secret to K8 git sync init container process.
### What type of PR is it?
Improvement
### Todos
* [ ] - Task
### How should this be tested?
* First time? Setup Travis CI as described on https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? Yes/No
* Is there breaking changes for older versions? Yes/No
* Does this needs documentation? Yes/No
Author: Manikandan R <ma...@gmail.com>
Closes #391 from manirajv06/SUBMARINE-597 and squashes the following commits:
162950c [Manikandan R] Fixed review comments (continuation of previous commit)
b29cc79 [Manikandan R] Fixed review comments
88d3f77 [Manikandan R] Fixed checkstyle issues
1ba4064 [Manikandan R] SUBMARINE-597: Support for "ssh" based git sync mode
---
.../experiment/codelocalizer/GitCodeLocalizer.java | 14 +++---
.../codelocalizer/SSHGitCodeLocalizer.java | 39 +++++++++++++++-
.../submitter/k8s/parser/ExperimentSpecParser.java | 47 ++++++++++++++-----
.../submitter/k8s/ExperimentSpecParserTest.java | 52 ++++++++++++++++++++++
.../server/submitter/k8s/SpecBuilder.java | 2 +
.../pytorch_job_req_ssh_git_code_localizer.json | 30 +++++++++++++
.../apache/submarine/rest/ExperimentRestApiIT.java | 7 +++
.../tf-mnist-with-ssh-git-code-localizer-req.json | 28 ++++++++++++
8 files changed, 200 insertions(+), 19 deletions(-)
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
index ddb3c78..efbcfda 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
@@ -19,8 +19,8 @@
package org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
@@ -86,16 +86,16 @@ public abstract class GitCodeLocalizer extends AbstractCodeLocalizer {
throws InvalidSpecException {
try {
- URL urlParser = new URL(url);
- String protocol = urlParser.getProtocol();
- if (protocol.equals(GitCodeLocalizerModes.HTTP.getMode())) {
+ URI uriParser = new URI(url);
+ String scheme = uriParser.getScheme();
+ if (scheme.equals(GitCodeLocalizerModes.HTTP.getMode())) {
return new HTTPGitCodeLocalizer(url);
- } else if (protocol.equals(GitCodeLocalizerModes.SSH.getMode())) {
+ } else if (scheme.equals(GitCodeLocalizerModes.SSH.getMode())) {
return new SSHGitCodeLocalizer(url);
} else {
return new DummyCodeLocalizer(url);
}
- } catch (MalformedURLException e) {
+ } catch (URISyntaxException e) {
throw new InvalidSpecException(
"Invalid Code Spec: URL is malformed. " + url);
}
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
index 1a8de83..085ae0b 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
@@ -19,17 +19,52 @@
package org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
+import java.util.List;
+
+import io.kubernetes.client.models.V1Container;
+import io.kubernetes.client.models.V1EnvVar;
import io.kubernetes.client.models.V1PodSpec;
+import io.kubernetes.client.models.V1SecurityContext;
+import io.kubernetes.client.models.V1VolumeMount;
public class SSHGitCodeLocalizer extends GitCodeLocalizer {
+ public static final String GIT_SECRET_NAME = "git-creds";
+ public static final int GIT_SECRET_MODE = 0400;
+ public static final String GIT_SECRET_MOUNT_NAME = "git-secret";
+ public static final String GIT_SECRET_PATH = "/etc/git-secret";
+ public static final long GIT_SYNC_USER = 65533L;
+ public static final String GIT_SYNC_SSH_NAME = "GIT_SYNC_SSH";
+ public static final String GIT_SYNC_SSH_VALUE = "true";
+
public SSHGitCodeLocalizer(String url) {
super(url);
}
@Override
public void localize(V1PodSpec podSpec) {
- // Code SSH based logic here
+ super.localize(podSpec);
+ for (V1Container container : podSpec.getInitContainers()) {
+ if (container.getName().equals(CODE_LOCALIZER_INIT_CONTAINER_NAME)) {
+ List<V1EnvVar> gitSyncEnvVars = container.getEnv();
+ V1EnvVar sshEnv = new V1EnvVar();
+ sshEnv.setName(GIT_SYNC_SSH_NAME);
+ sshEnv.setValue(GIT_SYNC_SSH_VALUE);
+ gitSyncEnvVars.add(sshEnv);
+
+ List<V1VolumeMount> mounts = container.getVolumeMounts();
+ V1VolumeMount mount = new V1VolumeMount();
+ mount.setName(GIT_SECRET_MOUNT_NAME);
+ mount.setMountPath(GIT_SECRET_PATH);
+ mount.setReadOnly(true);
+ mounts.add(mount);
+
+ V1SecurityContext containerSecurityContext =
+ new V1SecurityContext();
+ containerSecurityContext
+ .setRunAsUser(SSHGitCodeLocalizer.GIT_SYNC_USER);
+ container.setSecurityContext(containerSecurityContext);
+ }
+ }
}
-
}
diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
index eb19e04..e4190b7 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
@@ -23,9 +23,12 @@ import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.models.V1Container;
import io.kubernetes.client.models.V1EnvVar;
import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.models.V1PodSecurityContext;
import io.kubernetes.client.models.V1PodSpec;
import io.kubernetes.client.models.V1PodTemplateSpec;
import io.kubernetes.client.models.V1ResourceRequirements;
+import io.kubernetes.client.models.V1SecretVolumeSource;
+import io.kubernetes.client.models.V1Volume;
import io.kubernetes.client.models.V1VolumeMount;
import org.apache.submarine.commons.utils.SubmarineConfVars;
@@ -39,6 +42,7 @@ import org.apache.submarine.server.api.spec.EnvironmentSpec;
import org.apache.submarine.server.environment.EnvironmentManager;
import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.AbstractCodeLocalizer;
import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.CodeLocalizer;
+import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.SSHGitCodeLocalizer;
import org.apache.submarine.server.submitter.k8s.model.MLJob;
import org.apache.submarine.server.submitter.k8s.model.MLJobReplicaSpec;
import org.apache.submarine.server.submitter.k8s.model.MLJobReplicaType;
@@ -182,18 +186,41 @@ public class ExperimentSpecParser {
if (podSpec.getInitContainers() != null
&& podSpec.getInitContainers().size() > 0) {
- String volumeName = podSpec.getInitContainers().get(0).getVolumeMounts()
- .get(0).getName();
- String path = podSpec.getInitContainers().get(0).getVolumeMounts()
- .get(0).getMountPath();
-
- V1VolumeMount mount = new V1VolumeMount();
- mount.setName(volumeName);
- mount.setMountPath(path);
+ List<V1VolumeMount> initContainerVolumeMounts =
+ podSpec.getInitContainers().get(0).getVolumeMounts();
List<V1VolumeMount> volumeMounts = new ArrayList<V1VolumeMount>();
- volumeMounts.add(mount);
- container.setVolumeMounts(volumeMounts);
+
+ // Populate container volume mounts using Init container info
+ for (V1VolumeMount initContainerVolumeMount : initContainerVolumeMounts) {
+ String volumeName = initContainerVolumeMount.getName();
+ String path = initContainerVolumeMount.getMountPath();
+ if (volumeName
+ .equals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME)) {
+ V1VolumeMount mount = new V1VolumeMount();
+ mount.setName(volumeName);
+ mount.setMountPath(path);
+ volumeMounts.add(mount);
+ container.setVolumeMounts(volumeMounts);
+ } else if (volumeName
+ .equals(SSHGitCodeLocalizer.GIT_SECRET_MOUNT_NAME)) {
+ V1Volume volume = new V1Volume();
+ volume.setName(volumeName);
+
+ List<V1Volume> existingVolumes = podSpec.getVolumes();
+ V1SecretVolumeSource secret = new V1SecretVolumeSource();
+ secret.secretName(SSHGitCodeLocalizer.GIT_SECRET_NAME);
+ secret.setDefaultMode(SSHGitCodeLocalizer.GIT_SECRET_MODE);
+ volume.setSecret(secret);
+ existingVolumes.add(volume);
+
+ // Pod level security context
+ V1PodSecurityContext podSecurityContext =
+ new V1PodSecurityContext();
+ podSecurityContext.setFsGroup(SSHGitCodeLocalizer.GIT_SYNC_USER);
+ podSpec.setSecurityContext(podSecurityContext);
+ }
+ }
V1EnvVar codeEnvVar = new V1EnvVar();
codeEnvVar.setName(AbstractCodeLocalizer.CODE_LOCALIZER_PATH_ENV_VAR);
diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
index 18347dd..ae4af9f 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
@@ -46,6 +46,7 @@ import org.apache.submarine.server.submitter.k8s.model.tfjob.TFJobReplicaType;
import org.apache.submarine.server.submitter.k8s.parser.ExperimentSpecParser;
import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.AbstractCodeLocalizer;
import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.GitCodeLocalizer;
+import org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.SSHGitCodeLocalizer;
import org.junit.Assert;
import org.junit.Test;
import io.kubernetes.client.models.V1Container;
@@ -310,4 +311,55 @@ public class ExperimentSpecParserTest extends SpecBuilder {
Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME,
V1Volume.getName());
}
+
+ @Test
+ public void testValidPyTorchJobSpecWithSSHGitCodeLocalizer()
+ throws IOException, URISyntaxException, InvalidSpecException {
+ ExperimentSpec jobSpec =
+ (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
+ pytorchJobWithSSHGitCodeLocalizerFile);
+ PyTorchJob pyTorchJob = (PyTorchJob) ExperimentSpecParser.parseJob(jobSpec);
+
+ MLJobReplicaSpec mlJobReplicaSpec = pyTorchJob.getSpec().getReplicaSpecs()
+ .get(PyTorchJobReplicaType.Master);
+ Assert.assertEquals(1,
+ mlJobReplicaSpec.getTemplate().getSpec().getInitContainers().size());
+ V1Container initContainer =
+ mlJobReplicaSpec.getTemplate().getSpec().getInitContainers().get(0);
+ Assert.assertEquals(
+ AbstractCodeLocalizer.CODE_LOCALIZER_INIT_CONTAINER_NAME,
+ initContainer.getName());
+ Assert.assertEquals(GitCodeLocalizer.GIT_SYNC_IMAGE,
+ initContainer.getImage());
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME,
+ initContainer.getVolumeMounts().get(0).getName());
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_PATH,
+ initContainer.getVolumeMounts().get(0).getMountPath());
+ for (V1EnvVar env : initContainer.getEnv()) {
+ if (env.getName().equals(SSHGitCodeLocalizer.GIT_SYNC_SSH_NAME)) {
+ Assert.assertEquals(SSHGitCodeLocalizer.GIT_SYNC_SSH_VALUE,
+ env.getValue());
+ }
+ }
+
+ V1Container container =
+ mlJobReplicaSpec.getTemplate().getSpec().getInitContainers().get(0);
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME,
+ container.getVolumeMounts().get(0).getName());
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_PATH,
+ container.getVolumeMounts().get(0).getMountPath());
+ for (V1EnvVar env : container.getEnv()) {
+ if (env.getName()
+ .equals(AbstractCodeLocalizer.CODE_LOCALIZER_PATH_ENV_VAR)) {
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_PATH,
+ env.getValue());
+ }
+ }
+
+ V1Volume V1Volume =
+ mlJobReplicaSpec.getTemplate().getSpec().getVolumes().get(0);
+ Assert.assertEquals(new V1EmptyDirVolumeSource(), V1Volume.getEmptyDir());
+ Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME,
+ V1Volume.getName());
+ }
}
diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
index 0584276..d94a161 100644
--- a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
+++ b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
@@ -42,6 +42,8 @@ public abstract class SpecBuilder {
protected final String notebookReqFile = "/notebook_req.json";
protected final String pytorchJobWithHTTPGitCodeLocalizerFile =
"/pytorch_job_req_http_git_code_localizer.json";
+ protected final String pytorchJobWithSSHGitCodeLocalizerFile =
+ "/pytorch_job_req_ssh_git_code_localizer.json";
protected Object buildFromJsonFile(Object obj, String filePath) throws IOException,
URISyntaxException {
diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req_ssh_git_code_localizer.json b/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req_ssh_git_code_localizer.json
new file mode 100644
index 0000000..9afdec4
--- /dev/null
+++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req_ssh_git_code_localizer.json
@@ -0,0 +1,30 @@
+{
+ "meta": {
+ "name": "pytorch-dist-mnist",
+ "namespace": "submarine",
+ "framework": "PyTorch",
+ "cmd": "python /var/mnist.py --backend gloo",
+ "envVars": {
+ "ENV_1": "ENV1"
+ }
+ },
+ "environment": {
+ "image": "apache/submarine:pytorch-dist-mnist-1.0"
+ },
+ "spec": {
+ "Master": {
+ "name": "master",
+ "replicas": 1,
+ "resources": "cpu=2,memory=2048M"
+ },
+ "Worker": {
+ "name": "worker",
+ "replicas": 2,
+ "resources": "cpu=1,memory=1024M"
+ }
+ },
+ "code": {
+ "syncMode": "git",
+ "url" : "ssh://git@github.com/apache/submarine.git"
+ }
+}
diff --git a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
index e440d15..c83e055 100644
--- a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
+++ b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
@@ -180,6 +180,13 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
run(body, patchBody, "application/json");
}
+ @Test
+ public void testTensorFlowUsingSSHCodeWithJsonSpec() throws Exception {
+ String body = loadContent("tensorflow/tf-mnist-with-ssh-git-code-localizer-req.json");
+ String patchBody = loadContent("tensorflow/tf-mnist-with-ssh-git-code-localizer-req.json");
+ run(body, patchBody, "application/json");
+ }
+
private void run(String body, String patchBody, String contentType) throws Exception {
// create
LOG.info("Create training job by Job REST API");
diff --git a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-with-ssh-git-code-localizer-req.json b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-with-ssh-git-code-localizer-req.json
new file mode 100644
index 0000000..d7564b9
--- /dev/null
+++ b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-with-ssh-git-code-localizer-req.json
@@ -0,0 +1,28 @@
+{
+ "meta": {
+ "name": "tf-mnist-json",
+ "namespace": "default",
+ "framework": "TensorFlow",
+ "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150",
+ "envVars": {
+ "ENV_1": "ENV1"
+ }
+ },
+ "environment": {
+ "image": "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0"
+ },
+ "spec": {
+ "Ps": {
+ "replicas": 1,
+ "resources": "cpu=1,memory=512M"
+ },
+ "Worker": {
+ "replicas": 1,
+ "resources": "cpu=1,memory=512M"
+ }
+ },
+ "code": {
+ "syncMode": "git",
+ "url" : "ssh://git@github.com/apache/submarine.git"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org