You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2015/06/25 03:07:58 UTC
[25/50] [abbrv] hadoop git commit: YARN-3834. Scrub debug logging of
tokens during resource localization. Contributed by Chris Nauroth
YARN-3834. Scrub debug logging of tokens during resource localization. Contributed by Chris Nauroth
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c7a9d50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c7a9d50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c7a9d50
Branch: refs/heads/HDFS-7966
Commit: 6c7a9d502a633b5aca75c9798f19ce4a5729014e
Parents: c7d022b
Author: Xuan <xg...@apache.org>
Authored: Sun Jun 21 17:13:44 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Sun Jun 21 17:13:44 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../localizer/ResourceLocalizationService.java | 29 +++++++++++++++++++-
.../TestResourceLocalizationService.java | 10 +++++--
3 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d89c285..b50f490 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -312,6 +312,9 @@ Release 2.8.0 - UNRELEASED
YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet.
(Varun Saxena via devaraj)
+ YARN-3834. Scrub debug logging of tokens during resource localization.
+ (Chris Nauroth via xgong)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 54c31c2..d6e0903 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -1208,7 +1209,7 @@ public class ResourceLocalizationService extends CompositeService
if (LOG.isDebugEnabled()) {
for (Token<? extends TokenIdentifier> tk : credentials
.getAllTokens()) {
- LOG.debug(tk.getService() + " : " + tk.encodeToUrlString());
+ LOG.debug(tk + " : " + buildTokenFingerprint(tk));
}
}
if (UserGroupInformation.isSecurityEnabled()) {
@@ -1228,6 +1229,32 @@ public class ResourceLocalizationService extends CompositeService
}
+ /**
+ * Returns a fingerprint of a token. The fingerprint is suitable for use in
+ * logging, because it cannot be used to determine the secret. The
+ * fingerprint is built using the first 10 bytes of a SHA-256 hash of the
+ * string encoding of the token. The returned string contains the hex
+ * representation of each byte, delimited by a space.
+ *
+ * @param tk token
+ * @return token fingerprint
+ * @throws IOException if there is an I/O error
+ */
+ @VisibleForTesting
+ static String buildTokenFingerprint(Token<? extends TokenIdentifier> tk)
+ throws IOException {
+ char[] digest = DigestUtils.sha256Hex(tk.encodeToUrlString()).toCharArray();
+ StringBuilder fingerprint = new StringBuilder();
+ for (int i = 0; i < 10; ++i) {
+ if (i > 0) {
+ fingerprint.append(' ');
+ }
+ fingerprint.append(digest[2 * i]);
+ fingerprint.append(digest[2 * i + 1]);
+ }
+ return fingerprint.toString();
+ }
+
static class CacheCleanup extends Thread {
private final Dispatcher dispatcher;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index a02b2b0..c515506 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -2035,7 +2035,7 @@ public class TestResourceLocalizationService {
}
private static Container getMockContainer(ApplicationId appId, int id,
- String user) {
+ String user) throws IOException {
Container c = mock(Container.class);
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(appId, 1);
@@ -2043,7 +2043,13 @@ public class TestResourceLocalizationService {
when(c.getUser()).thenReturn(user);
when(c.getContainerId()).thenReturn(cId);
Credentials creds = new Credentials();
- creds.addToken(new Text("tok" + id), getToken(id));
+ Token<? extends TokenIdentifier> tk = getToken(id);
+ String fingerprint = ResourceLocalizationService.buildTokenFingerprint(tk);
+ assertNotNull(fingerprint);
+ assertTrue(
+ "Expected token fingerprint of 10 hex bytes delimited by space.",
+ fingerprint.matches("^(([0-9a-f]){2} ){9}([0-9a-f]){2}$"));
+ creds.addToken(new Text("tok" + id), tk);
when(c.getCredentials()).thenReturn(creds);
when(c.toString()).thenReturn(cId.toString());
return c;