You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/02/06 23:52:28 UTC
tez git commit: TEZ-900. Confusing message for incorrect queue for
some tez examples. (Chang Li via hitesh)
Repository: tez
Updated Branches:
refs/heads/master 2b12f4cba -> 2b5eeea52
TEZ-900. Confusing message for incorrect queue for some tez examples. (Chang Li via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2b5eeea5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2b5eeea5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2b5eeea5
Branch: refs/heads/master
Commit: 2b5eeea52b63ee10aa92ce1d7cd7ac88aff6a730
Parents: 2b12f4c
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Feb 6 14:35:38 2015 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Feb 6 14:35:38 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/TezYarnClient.java | 8 ++++-
.../org/apache/tez/client/TestTezClient.java | 8 +++++
.../java/org/apache/tez/test/TestTezJobs.java | 35 ++++++++++++++++++++
4 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ab46d64..584f49a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -180,6 +180,7 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-900. Confusing message for incorrect queue for some tez examples.
TEZ-2036. OneToOneEdgeManager should enforce that source and destination
tasks have same number
TEZ-1895. Vertex reRunning should decrease successfulMembers of VertexGroupInfo.
http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index d58a0bb..eae1909 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -66,7 +67,12 @@ public class TezYarnClient extends FrameworkClient {
@Override
public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext) throws YarnException, IOException {
- return yarnClient.submitApplication(appSubmissionContext);
+ ApplicationId appId= yarnClient.submitApplication(appSubmissionContext);
+ ApplicationReport appReport = getApplicationReport(appId);
+ if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED){
+ throw new IOException("Failed to submit application to YARN: " + appReport.getDiagnostics());
+ }
+ return appId;
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index f4d4112..14d4d7f 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -141,6 +141,8 @@ public class TestTezClient {
TezClientForTest client = configure(lrs, isSession);
ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
client.start();
verify(client.mockYarnClient, times(1)).init((Configuration)any());
verify(client.mockYarnClient, times(1)).start();
@@ -197,6 +199,8 @@ public class TestTezClient {
when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId())
.thenReturn(appId2);
+ when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
dag = DAG.create("DAG").addVertex(
Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)));
dagClient = client.submitDAG(dag);
@@ -269,6 +273,8 @@ public class TestTezClient {
public void testMultipleSubmissionsJob(boolean isSession) throws Exception {
TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession);
+ when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
client1.start();
String mockLR1Name = "LR1";
@@ -287,6 +293,8 @@ public class TestTezClient {
client1.submitDAG(dag);
TezClientForTest client2 = configure();
+ when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
client2.start();
// verify resubmission of same dag to new client (simulates submission error resulting in the
http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index adf27e8..b9cb491 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -672,6 +672,41 @@ public class TestTezJobs {
}
+ @Test(timeout = 60000)
+ public void testInvalidQueueSubmission() throws Exception {
+
+ TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ try {
+
+ yarnClient.init(mrrTezCluster.getConfig());
+ yarnClient.start();
+
+ SimpleSessionExample job = new SimpleSessionExample();
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
+ tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");
+
+ String[] inputPaths = new String[1];
+ String[] outputPaths = new String[1];
+ String inputDirStr = "/tmp/owc-input";
+ inputPaths[0] = inputDirStr;
+ Path inputDir = new Path(inputDirStr);
+ remoteFs.mkdirs(inputDir);
+ String outputDirStr = "/tmp/owc-output";
+ outputPaths[0] = outputDirStr;
+ job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
+ StringUtils.join(",", outputPaths), "2" }, null);
+ fail("Job submission should have thrown an exception");
+ } catch(IOException e) {
+ Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN"));
+ } finally {
+ if (yarnClient != null) {
+ yarnClient.stop();
+ }
+ }
+
+ }
+
@Test (timeout=60000)
public void testVertexOrder() throws Exception {
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());