You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by zh...@apache.org on 2017/03/28 19:47:49 UTC
[08/50] [abbrv] tez git commit: TEZ-3550. Provide access to
sessionId/dagId via DagClient. (sseth)
TEZ-3550. Provide access to sessionId/dagId via DagClient. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cd44c906
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cd44c906
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cd44c906
Branch: refs/heads/TEZ-1190
Commit: cd44c906f762c24a13ddabe691e55c879c223eac
Parents: e1b0b28
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Feb 14 10:46:18 2017 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Feb 14 10:46:18 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../java/org/apache/tez/dag/api/client/DAGClient.java | 13 +++++++++++++
.../org/apache/tez/dag/api/client/DAGClientImpl.java | 10 ++++++++++
.../apache/tez/dag/api/client/DAGClientInternal.java | 14 +++++++++++++-
.../tez/dag/api/client/DAGClientTimelineImpl.java | 11 ++++++++++-
.../tez/dag/api/client/rpc/DAGClientRPCImpl.java | 10 +++++++++-
.../java/org/apache/tez/client/TestTezClient.java | 5 ++++-
.../apache/tez/dag/api/client/rpc/TestDAGClient.java | 2 ++
.../org/apache/tez/dag/api/client/MRDAGClient.java | 10 ++++++++++
9 files changed, 73 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6991f05..337b394 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3550. Provide access to sessionId/dagId via DagClient.
TEZ-3267. Publish queue name to ATS as part of dag summary.
TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
@@ -201,6 +202,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3550. Provide access to sessionId/dagId via DagClient.
TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
index 9b11b96..c70da75 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -84,6 +84,19 @@ public abstract class DAGClient implements Closeable {
throws IOException, TezException;
/**
+ * Get the dag identifier for the currently executing dag. This is a string
+ * which represents this dag
+ * @return the dag identifier
+ */
+ public abstract String getDagIdentifierString();
+
+ /**
+ * Get the session identifier for the session in which this dag is running
+ * @return the session identifier
+ */
+ public abstract String getSessionIdentifierString();
+
+ /**
* Kill a running DAG
*
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
index 4820b6e..1cf0bfc 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
@@ -319,6 +319,16 @@ public class DAGClientImpl extends DAGClient {
}
@Override
+ public String getDagIdentifierString() {
+ return realClient.getDagIdentifierString();
+ }
+
+ @Override
+ public String getSessionIdentifierString() {
+ return realClient.getSessionIdentifierString();
+ }
+
+ @Override
public void tryKillDAG() throws IOException, TezException {
if (!dagCompleted) {
realClient.tryKillDAG();
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java
index bb236a3..a3c898a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java
@@ -25,7 +25,6 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.tez.dag.api.TezException;
@@ -85,6 +84,19 @@ public abstract class DAGClientInternal implements Closeable {
throws IOException, TezException, ApplicationNotFoundException;
/**
+ * Get the dag identifier for the currently executing dag. This is a string
+ * which represents this dag
+ * @return the dag identifier
+ */
+ public abstract String getDagIdentifierString();
+
+ /**
+ * Get the session identifier for the session in which this dag is running
+ * @return the session identifier
+ */
+ public abstract String getSessionIdentifierString();
+
+ /**
* Kill a running DAG
*
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
index ffd91b7..d34dbf0 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
@@ -38,7 +38,6 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -182,6 +181,16 @@ public class DAGClientTimelineImpl extends DAGClientInternal {
}
@Override
+ public String getDagIdentifierString() {
+ return dagId;
+ }
+
+ @Override
+ public String getSessionIdentifierString() {
+ return appId.toString();
+ }
+
+ @Override
public void tryKillDAG() throws IOException, TezException {
throw new TezException("tryKillDAG is unsupported for DAGClientTimelineImpl");
}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
index 9eb9807..02935df 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -41,7 +41,6 @@ import org.apache.tez.dag.api.DAGNotRunningException;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.DagStatusSource;
import org.apache.tez.dag.api.client.StatusGetOpts;
@@ -130,6 +129,15 @@ public class DAGClientRPCImpl extends DAGClientInternal {
return null;
}
+ @Override
+ public String getDagIdentifierString() {
+ return dagId.toString();
+ }
+
+ @Override
+ public String getSessionIdentifierString() {
+ return appId.toString();
+ }
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index c9415a4..89310df 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -277,7 +277,8 @@ public class TestTezClient {
DAGClient dagClient = client.submitDAG(dag);
assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
-
+ assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());
+
if (isSession) {
verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any());
@@ -316,6 +317,7 @@ public class TestTezClient {
// same app master
verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
+ assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());
// additional resource is sent
ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
verify(client.sessionAmProxy, times(2)).submitDAG((RpcController)any(), captor1.capture());
@@ -325,6 +327,7 @@ public class TestTezClient {
} else {
// new app master
assertTrue(dagClient.getExecutionContext().contains(appId2.toString()));
+ assertEquals(dagClient.getSessionIdentifierString(), appId2.toString());
verify(client.mockYarnClient, times(2)).submitApplication(captor.capture());
// additional resource is added
ApplicationSubmissionContext context = captor.getValue();
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
index e979f8b..70ee1d4 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
@@ -213,6 +213,8 @@ public class TestDAGClient {
@Test(timeout = 5000)
public void testApp() throws IOException, TezException, ServiceException{
assertTrue(dagClient.getExecutionContext().contains(mockAppId.toString()));
+ assertEquals(mockAppId.toString(), dagClient.getSessionIdentifierString());
+ assertEquals(dagIdStr, dagClient.getDagIdentifierString());
DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient();
assertEquals(mockAppReport, realClient.getApplicationReportInternal());
}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd44c906/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java
index d743feb..42b52e0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java
@@ -66,6 +66,16 @@ public class MRDAGClient extends DAGClient {
}
@Override
+ public String getDagIdentifierString() {
+ return realClient.getDagIdentifierString();
+ }
+
+ @Override
+ public String getSessionIdentifierString() {
+ return realClient.getSessionIdentifierString();
+ }
+
+ @Override
public void tryKillDAG() throws IOException, TezException {
realClient.tryKillDAG();
}