You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zt...@apache.org on 2019/08/15 05:19:09 UTC
[hadoop] branch trunk updated: SUBMARINE-45. Can't specify queue by
using the parameter --queue. Contributed by Ayush Saxena, Zac Zhou.
This is an automated email from the ASF dual-hosted git repository.
ztang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 85a00fd SUBMARINE-45. Can't specify queue by using the parameter --queue. Contributed by Ayush Saxena, Zac Zhou.
85a00fd is described below
commit 85a00fd2bbb8e8fee5691eff78477cdcb746a926
Author: Zhankun Tang <zt...@apache.org>
AuthorDate: Thu Aug 15 13:18:29 2019 +0800
SUBMARINE-45. Can't specify queue by using the parameter --queue. Contributed by Ayush Saxena, Zac Zhou.
---
.../yarn/submarine/runtimes/yarnservice/AbstractServiceSpec.java | 1 +
.../yarn/submarine/client/cli/yarnservice/ParamBuilderForTest.java | 6 ++++++
.../submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java | 6 ++++++
.../client/cli/yarnservice/TestYarnServiceRunJobCliCommons.java | 1 +
4 files changed, 14 insertions(+)
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/AbstractServiceSpec.java b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/AbstractServiceSpec.java
index 245226c..3790ffe 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/AbstractServiceSpec.java
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/AbstractServiceSpec.java
@@ -78,6 +78,7 @@ public abstract class AbstractServiceSpec implements ServiceSpec {
serviceSpec.setName(parameters.getName());
serviceSpec.setVersion(String.valueOf(System.currentTimeMillis()));
serviceSpec.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
+ serviceSpec.setQueue(parameters.getQueue());
KerberosPrincipal kerberosPrincipal = KerberosPrincipalFactory
.create(fsOperations, remoteDirectoryManager, parameters);
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/ParamBuilderForTest.java b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/ParamBuilderForTest.java
index f4d4408..f4c8031 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/ParamBuilderForTest.java
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/ParamBuilderForTest.java
@@ -57,6 +57,12 @@ class ParamBuilderForTest {
return this;
}
+ ParamBuilderForTest withQueue(String queue) {
+ params.add("--queue");
+ params.add(queue);
+ return this;
+ }
+
ParamBuilderForTest withNumberOfWorkers(int numWorkers) {
params.add("--num_workers");
params.add(String.valueOf(numWorkers));
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
index 75bab04..cde7405 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
@@ -65,6 +65,7 @@ import static org.apache.hadoop.yarn.submarine.client.cli.yarnservice.TestYarnSe
import static org.apache.hadoop.yarn.submarine.client.cli.yarnservice.TestYarnServiceRunJobCliCommons.DEFAULT_WORKER_DOCKER_IMAGE;
import static org.apache.hadoop.yarn.submarine.client.cli.yarnservice.TestYarnServiceRunJobCliCommons.DEFAULT_WORKER_LAUNCH_CMD;
import static org.apache.hadoop.yarn.submarine.client.cli.yarnservice.TestYarnServiceRunJobCliCommons.DEFAULT_WORKER_RESOURCES;
+import static org.apache.hadoop.yarn.submarine.client.cli.yarnservice.TestYarnServiceRunJobCliCommons.DEFAULT_QUEUE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -134,6 +135,7 @@ public class TestYarnServiceRunJobCli {
assertEquals(DEFAULT_WORKER_DOCKER_IMAGE, workerComp.getArtifact().getId());
assertEquals(DEFAULT_PS_DOCKER_IMAGE, psComp.getArtifact().getId());
+ assertEquals(DEFAULT_QUEUE, serviceSpec.getQueue());
assertTrue(SubmarineLogs.isVerbose());
}
@@ -183,6 +185,7 @@ public class TestYarnServiceRunJobCli {
.withDockerImage(DEFAULT_DOCKER_IMAGE)
.withInputPath(DEFAULT_INPUT_PATH)
.withCheckpointPath(DEFAULT_CHECKPOINT_PATH)
+ .withQueue(DEFAULT_QUEUE)
.withNumberOfWorkers(3)
.withWorkerDockerImage(DEFAULT_WORKER_DOCKER_IMAGE)
.withWorkerLaunchCommand(DEFAULT_WORKER_LAUNCH_CMD)
@@ -219,6 +222,7 @@ public class TestYarnServiceRunJobCli {
.withDockerImage(DEFAULT_DOCKER_IMAGE)
.withInputPath(DEFAULT_INPUT_PATH)
.withCheckpointPath(DEFAULT_CHECKPOINT_PATH)
+ .withQueue(DEFAULT_QUEUE)
.withNumberOfWorkers(3)
.withWorkerDockerImage(DEFAULT_WORKER_DOCKER_IMAGE)
.withWorkerLaunchCommand(DEFAULT_WORKER_LAUNCH_CMD)
@@ -562,6 +566,7 @@ public class TestYarnServiceRunJobCli {
.withDockerImage(DEFAULT_DOCKER_IMAGE)
.withInputPath(DEFAULT_INPUT_PATH)
.withCheckpointPath(DEFAULT_CHECKPOINT_PATH)
+ .withQueue(DEFAULT_QUEUE)
.withNumberOfWorkers(3)
.withWorkerDockerImage(DEFAULT_WORKER_DOCKER_IMAGE)
.withWorkerLaunchCommand(DEFAULT_WORKER_LAUNCH_CMD)
@@ -602,6 +607,7 @@ public class TestYarnServiceRunJobCli {
.withDockerImage(DEFAULT_DOCKER_IMAGE)
.withInputPath(DEFAULT_INPUT_PATH)
.withCheckpointPath(DEFAULT_CHECKPOINT_PATH)
+ .withQueue(DEFAULT_QUEUE)
.withNumberOfWorkers(3)
.withWorkerDockerImage(DEFAULT_WORKER_DOCKER_IMAGE)
.withWorkerLaunchCommand(DEFAULT_WORKER_LAUNCH_CMD)
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliCommons.java b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliCommons.java
index 94e2c37..40e047b 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliCommons.java
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliCommons.java
@@ -40,6 +40,7 @@ public class TestYarnServiceRunJobCliCommons {
static final String DEFAULT_DOCKER_IMAGE = "tf-docker:1.1.0";
static final String DEFAULT_INPUT_PATH = "s3://input";
static final String DEFAULT_CHECKPOINT_PATH = "s3://output";
+ static final String DEFAULT_QUEUE = "root.queue";
static final String DEFAULT_WORKER_DOCKER_IMAGE = "worker.image";
static final String DEFAULT_PS_DOCKER_IMAGE = "ps.image";
static final String DEFAULT_WORKER_LAUNCH_CMD = "python run-job.py";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org