You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2016/10/07 21:53:47 UTC
tez git commit: TEZ-3335. DAG client thinks app is still running when
app status is null (jeagles)
Repository: tez
Updated Branches:
refs/heads/master dceb365c2 -> 539b8e710
TEZ-3335. DAG client thinks app is still running when app status is null (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/539b8e71
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/539b8e71
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/539b8e71
Branch: refs/heads/master
Commit: 539b8e710943b0b79362afc61e246bb83e862ac5
Parents: dceb365
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Fri Oct 7 16:53:22 2016 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Fri Oct 7 16:53:22 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 5 +++-
.../org/apache/tez/client/TezYarnClient.java | 12 +++++++++-
.../org/apache/tez/client/TestTezClient.java | 25 ++++++++++++++++++++
3 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 63eda94..addd2c0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3335. DAG client thinks app is still running when app status is null
TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317.
TEZ-3460. Fix precommit release audit warning.
TEZ-3368. NPE in DelayedContainerManager
@@ -117,7 +118,8 @@ Release 0.8.5: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
-
+
+ TEZ-3335. DAG client thinks app is still running when app status is null
TEZ-3460. Fix precommit release audit warning.
TEZ-3368. NPE in DelayedContainerManager
TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs
@@ -611,6 +613,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3335. DAG client thinks app is still running when app status is null
TEZ-3460. Fix precommit release audit warning.
TEZ-3368. NPE in DelayedContainerManager
TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs
http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index 241e15c..3ac82ac 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
@@ -86,6 +87,15 @@ public class TezYarnClient extends FrameworkClient {
@Override
public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException {
- return yarnClient.getApplicationReport(appId);
+ ApplicationReport report = yarnClient.getApplicationReport(appId);
+ if (report.getYarnApplicationState() == null) {
+ // The state can be null when the ResourceManager does not know about the app but the YARN
+ // application history server has an incomplete entry for it. Treat this scenario as if the
+ // application does not exist, since the final app status cannot be determined. This also
+ // matches the behavior for this scenario if the history server was not configured.
+ throw new ApplicationNotFoundException("YARN reports no state for application "
+ + appId);
+ }
+ return report;
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 51f36a3..48dfff4 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -48,6 +48,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.protobuf.ServiceException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
@@ -55,6 +56,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -63,6 +65,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.counters.LimitExceededException;
import org.apache.tez.common.counters.Limits;
@@ -147,6 +151,8 @@ public class TestTezClient {
ApplicationId appId1 = ApplicationId.newInstance(0, 1);
YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS);
when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
+ when(yarnClient.getApplicationReport(appId1).getYarnApplicationState()).thenReturn(YarnApplicationState.NEW);
+ when(yarnClient.submitApplication(any(ApplicationSubmissionContext.class))).thenReturn(appId1);
DAGClientAMProtocolBlockingPB sessionAmProxy = mock(DAGClientAMProtocolBlockingPB.class, RETURNS_DEEP_STUBS);
when(sessionAmProxy.getAMStatus(any(RpcController.class), any(GetAMStatusRequestProto.class)))
@@ -769,4 +775,23 @@ public class TestTezClient {
}
client.stop();
}
+
+ @Test(timeout = 10000)
+ public void testMissingYarnAppStatus() throws Exception {
+ // verify an app not found exception is thrown when YARN reports a null app status
+ ApplicationId appId1 = ApplicationId.newInstance(0, 1);
+ ApplicationReport mockReport = mock(ApplicationReport.class);
+ when(mockReport.getApplicationId()).thenReturn(appId1);
+ when(mockReport.getYarnApplicationState()).thenReturn(null);
+ YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS);
+ when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
+ when(yarnClient.getApplicationReport(appId1)).thenReturn(mockReport);
+ TezYarnClient tezClient = new TezYarnClient(yarnClient);
+ tezClient.init(new TezConfiguration(false), new YarnConfiguration());
+ try {
+ tezClient.getApplicationReport(appId1);
+ fail("getApplicationReport should have thrown");
+ } catch (ApplicationNotFoundException e) {
+ }
+ }
}