You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2015/10/10 04:36:13 UTC
tez git commit: TEZ-2874. Improved logging for caller context.
(hitesh)
Repository: tez
Updated Branches:
refs/heads/master 8a1ac4458 -> 327e02a76
TEZ-2874. Improved logging for caller context. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/327e02a7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/327e02a7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/327e02a7
Branch: refs/heads/master
Commit: 327e02a761eb01c74acdf5b7af65f79e23b4e28b
Parents: 8a1ac44
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Oct 9 19:35:05 2015 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Oct 9 19:36:02 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../java/org/apache/tez/client/CallerContext.java | 13 +++++++++++--
.../main/java/org/apache/tez/client/TezClient.java | 17 +++++++++++++----
.../src/main/java/org/apache/tez/dag/api/DAG.java | 5 +++++
.../test/java/org/apache/tez/dag/api/TestDAG.java | 15 ++++++++-------
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +-
6 files changed, 40 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1ced5d8..32aca94 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.8.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2874. Improved logging for caller context.
TEZ-2875. Enable missing tests in TestAnalyzer
TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
TEZ-1788. Allow vertex level disabling of speculation
@@ -202,6 +203,7 @@ Release 0.7.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES
+ TEZ-2874. Improved logging for caller context.
TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
TEZ-2868. Fix setting Caller Context in Tez Examples.
TEZ-2860. NPE in DAGClientImpl.
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/client/CallerContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java
index ba68851..809cf47 100644
--- a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java
+++ b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java
@@ -162,10 +162,19 @@ public class CallerContext {
@Override
public String toString() {
- return "context=" + context
+ return "{ context=" + context
+ ", callerType=" + callerType
+ ", callerId=" + callerId
- + ", blob=" + blob;
+ + ", blob=" + blob
+ + " }";
}
+ public String contextAsSimpleString() {
+ return "{ context=" + context
+ + ", callerType=" + callerType
+ + ", callerId=" + callerId
+ + " }";
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 0c50d86..ff131e0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -57,7 +57,6 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezReflectionException;
-import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAMStatusRequestProto;
@@ -464,10 +463,15 @@ public class TezClient {
verifySessionStateForSubmission();
String dagId = null;
+ String callerContextStr = "";
+ if (dag.getCallerContext() != null) {
+ callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
+ }
LOG.info("Submitting dag to TezSession"
+ ", sessionName=" + clientName
+ ", applicationId=" + sessionAppId
- + ", dagName=" + dag.getName());
+ + ", dagName=" + dag.getName()
+ + callerContextStr);
if (!additionalLocalResources.isEmpty()) {
for (LocalResource lr : additionalLocalResources.values()) {
@@ -868,13 +872,18 @@ public class TezClient {
// Add credentials for tez-local resources.
Map<String, LocalResource> tezJarResources = getTezJarResources(credentials);
ApplicationSubmissionContext appContext = TezClientUtils
- .createApplicationSubmissionContext(
+ .createApplicationSubmissionContext(
appId, dag, dag.getName(), amConfig, tezJarResources, credentials,
usingTezArchiveDeploy, apiVersionInfo, historyACLPolicyManager,
servicePluginsDescriptor, javaOptsChecker);
+ String callerContextStr = "";
+ if (dag.getCallerContext() != null) {
+ callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
+ }
LOG.info("Submitting DAG to YARN"
+ ", applicationId=" + appId
- + ", dagName=" + dag.getName());
+ + ", dagName=" + dag.getName()
+ + callerContextStr);
frameworkClient.submitApplication(appContext);
ApplicationReport appReport = frameworkClient.getApplicationReport(appId);
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index e4fdc18..c02d86f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -1084,4 +1084,9 @@ public class DAG {
}
return false;
}
+
+ public synchronized CallerContext getCallerContext() {
+ return this.callerContext;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
index 24c20b5..839e780 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
@@ -311,24 +311,25 @@ public class TestDAG {
@Test
public void testCallerContext() {
- DAG dag = DAG.create("dag1");
try {
- CallerContext callerContext = CallerContext.create("ctxt", "", "", "desc");
+ CallerContext.create("ctxt", "", "", "desc");
Assert.fail("Expected failure for invalid args");
} catch (Exception e) {
// Expected
}
try {
- CallerContext callerContext = CallerContext.create("", "desc");
+ CallerContext.create("", "desc");
Assert.fail("Expected failure for invalid args");
} catch (Exception e) {
// Expected
}
- CallerContext callerContext;
- callerContext = CallerContext.create("ctxt", "a", "a", "desc");
- callerContext = CallerContext.create("ctxt", "desc");
- callerContext = CallerContext.create("ctxt", null);
+ CallerContext.create("ctxt", "a", "a", "desc");
+ CallerContext.create("ctxt", null);
+
+ CallerContext callerContext = CallerContext.create("ctxt", "desc");
+ Assert.assertTrue(callerContext.toString().contains("desc"));
+ Assert.assertFalse(callerContext.contextAsSimpleString().contains("desc"));
}
http://git-wip-us.apache.org/repos/asf/tez/blob/327e02a7/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index e165397..d774b9a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2308,7 +2308,7 @@ public class DAGAppMaster extends AbstractService {
if (dagPlan.hasCallerContext()) {
CallerContext callerContext = DagTypeConverters.convertCallerContextFromProto(
dagPlan.getCallerContext());
- callerContextStr = ", callerContext=" + callerContext.toString();
+ callerContextStr = ", callerContext=" + callerContext.contextAsSimpleString();
}
LOG.info("Running DAG: " + dagPlan.getName() + callerContextStr);