You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2020/02/17 08:05:01 UTC
[tez] branch master updated: TEZ-4109: Improve TezCommonUtils.getCredentialsInfo and use it from more contexts (László Bodor reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 9be1b1c TEZ-4109: Improve TezCommonUtils.getCredentialsInfo and use it from more contexts (László Bodor reviewed by Ashutosh Chauhan)
9be1b1c is described below
commit 9be1b1cbc6e859ae815e4f0629e9103d67a1b76b
Author: László Bodor <bo...@gmail.com>
AuthorDate: Mon Feb 17 09:01:57 2020 +0100
TEZ-4109: Improve TezCommonUtils.getCredentialsInfo and use it from more contexts (László Bodor reviewed by Ashutosh Chauhan)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../main/java/org/apache/tez/client/TezClient.java | 3 +--
.../java/org/apache/tez/client/TezClientUtils.java | 3 ++-
.../java/org/apache/tez/common/TezCommonUtils.java | 21 +++++++++++----------
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 4 ++--
.../tez/service/impl/ContainerRunnerImpl.java | 2 ++
.../tez/runtime/task/TaskRunner2Callable.java | 2 ++
.../java/org/apache/tez/runtime/task/TezChild.java | 3 +++
7 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index a615179..60c0e5e 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.NumberFormat;
-import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -1127,7 +1126,7 @@ public class TezClient {
+ ", applicationId=" + appId
+ ", dagName=" + dag.getName()
+ callerContextStr);
-
+ TezCommonUtils.logCredentials(LOG, credentials, "appContext");
frameworkClient.submitApplication(appContext);
ApplicationReport appReport = frameworkClient.getApplicationReport(appId);
LOG.info("The url to track the Tez AM: " + appReport.getTrackingUrl());
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index bcb0920..db1bc0c 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -398,7 +398,6 @@ public class TezClientUtils {
Objects.requireNonNull(sessionCredentials);
TezCommonUtils.logCredentials(LOG, sessionCredentials, "session");
-
Credentials dagCredentials = new Credentials();
// All session creds are required for the DAG.
dagCredentials.mergeAll(sessionCredentials);
@@ -725,6 +724,7 @@ public class TezClientUtils {
if (amConfig.getCredentials() != null) {
amLaunchCredentials.mergeAll(amConfig.getCredentials());
}
+ TezCommonUtils.logCredentials(LOG, amLaunchCredentials, "amLaunch");
return amLaunchCredentials;
}
@@ -750,6 +750,7 @@ public class TezClientUtils {
JavaOptsChecker javaOptsChecker) throws IOException {
Credentials dagCredentials = setupDAGCredentials(dag, credentials,
amConfig.getTezConfiguration());
+ TezCommonUtils.logCredentials(LOG, dagCredentials, "dagPlan");
return dag.createDag(amConfig.getTezConfiguration(), dagCredentials, tezJarResources,
amConfig.getBinaryConfLR(), tezLrsAsArchive, servicePluginsDescriptor, javaOptsChecker);
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index 16165e2..2725958 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -22,10 +22,10 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
+import java.util.stream.Collectors;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.Inflater;
@@ -405,20 +405,21 @@ public class TezCommonUtils {
}
public static String getCredentialsInfo(Credentials credentials, String identifier) {
+ if (credentials == null) {
+ return "Credentials: #" + identifier + "Tokens=null";
+ }
+
StringBuilder sb = new StringBuilder();
sb.append("Credentials: #" + identifier + "Tokens=").append(credentials.numberOfTokens());
if (credentials.numberOfTokens() > 0) {
sb.append(", Services=");
- Iterator<Token<?>> tokenItr = credentials.getAllTokens().iterator();
- if (tokenItr.hasNext()) {
- Token token = tokenItr.next();
- sb.append(token.getService()).append("(").append(token.getKind()).append(")");
+ sb.append(credentials.getAllTokens().stream()
+ .map(t -> String.format("%s(%s)", t.getService(), t.getKind()))
+ .collect(Collectors.joining(",")));
- }
- while(tokenItr.hasNext()) {
- Token token = tokenItr.next();
- sb.append(",").append(token.getService()).append("(").append(token.getKind()).append(")");
- }
+ sb.append(", TokenDetails=");
+ sb.append(credentials.getAllTokens().stream().map(Token::toString)
+ .collect(Collectors.joining(",")));
}
return sb.toString();
}
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 99b01d6..18cffad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -374,7 +374,7 @@ public class DAGAppMaster extends AbstractService {
LOG.info("Created DAGAppMaster for application " + applicationAttemptId
+ ", versionInfo=" + dagVersionInfo.toString());
-
+ TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
// Pull this WebAppUtils function into Tez until YARN-4186
@@ -1020,7 +1020,7 @@ public class DAGAppMaster extends AbstractService {
// TODO Does this move to the client in case of work-preserving recovery.
TokenCache.setSessionToken(sessionToken, dagCredentials);
-
+ TezCommonUtils.logCredentials(LOG, dagCredentials, "newDag");
// create single dag
DAGImpl newDag =
new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(),
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index e7777e2..eef73a0 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -266,6 +266,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
// TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString());
ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
+ TezCommonUtils.logCredentials(LOG, credentials, "taskCallable");
TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()),
new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs,
workingDir, credentials, memoryPerExecutor, sharedExecutor);
@@ -457,6 +458,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
new AtomicLong(0),
request.getContainerIdString());
+ TezCommonUtils.logCredentials(LOG, taskUgi.getCredentials(), "taskUgi");
taskRunner = new TezTaskRunner2(conf, taskUgi, localDirs,
ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()),
request.getAppAttemptNumber(),
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java
index b39af69..0e6dfda 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java
@@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.tez.common.CallableWithNdc;
+import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
@@ -66,6 +67,7 @@ public class TaskRunner2Callable extends CallableWithNdc<TaskRunner2Callable.Tas
}
LOG.info("Initializing task" + ", taskAttemptId={}", task.getTaskAttemptID());
TezUtilsInternal.setHadoopCallerContext(task.getHadoopShim(), task.getTaskAttemptID());
+ TezCommonUtils.logCredentials(LOG, ugi.getCredentials(), "taskInit");
task.initialize();
if (!stopRequested.get() && !Thread.currentThread().isInterrupted()) {
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 0693317..6dd146a 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -198,6 +198,7 @@ public class TezChild {
this.umbilical = umbilical;
ownUmbilical = false;
}
+ TezCommonUtils.logCredentials(LOG, credentials, "tezChildInit");
}
public ContainerExecutionResult run() throws IOException, InterruptedException, TezException {
@@ -237,6 +238,7 @@ public class TezChild {
shutdown();
}
}
+ TezCommonUtils.logCredentials(LOG, containerTask.getCredentials(), "containerTask");
if (containerTask.shouldDie()) {
LOG.info("ContainerTask returned shouldDie=true for container {}, Exiting", containerIdString);
shutdown();
@@ -256,6 +258,7 @@ public class TezChild {
FileSystem.clearStatistics();
childUGI = handleNewTaskCredentials(containerTask, childUGI);
+ TezCommonUtils.logCredentials(LOG, childUGI.getCredentials(), "taskChildUGI");
handleNewTaskLocalResources(containerTask, childUGI);
cleanupOnTaskChanged(containerTask);