You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2016/02/27 21:09:27 UTC
hadoop git commit: HADOOP-12825. Log slow name resolutions. (Sidharta
Seethana via stevel)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 d55863de0 -> 621800859
HADOOP-12825. Log slow name resolutions. (Sidharta Seethana via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62180085
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62180085
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62180085
Branch: refs/heads/branch-2.8
Commit: 6218008591f2ecdf16cb73d77492dd227d16b9cd
Parents: d55863d
Author: Steve Loughran <st...@apache.org>
Authored: Sat Feb 27 20:05:35 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Feb 27 20:05:35 2016 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/fs/CommonConfigurationKeys.java | 14 ++++++-
.../apache/hadoop/security/SecurityUtil.java | 42 +++++++++++++++++++-
.../src/main/resources/core-default.xml | 18 +++++++++
4 files changed, 74 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62180085/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 44d8781..6b2c2ee 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -432,6 +432,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12711. Remove dependency on commons-httpclient for ServletUtil
(Wei-Chiu Chuang via iwasakims)
+ HADOOP-12825. Log slow name resolutions.
+ (Sidharta Seethana via stevel)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62180085/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index fcc3988..9726356 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -188,7 +188,19 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"hadoop.security.token.service.use_ip";
public static final boolean HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT =
true;
-
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ public static final String HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY =
+ "hadoop.security.dns.log-slow-lookups.enabled";
+ public static final boolean
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT = false;
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ public static final String
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY =
+ "hadoop.security.dns.log-slow-lookups.threshold.ms";
+ public static final int
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT = 1000;
+
/**
* HA health monitor and failover controller.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62180085/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 38096ab..61cd516 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -30,6 +30,7 @@ import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.List;
import java.util.ServiceLoader;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -47,6 +48,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.util.StopWatch;
import org.apache.hadoop.util.StringUtils;
@@ -79,6 +81,9 @@ public class SecurityUtil {
setTokenServiceUseIp(useIp);
}
+ private static boolean logSlowLookups = getLogSlowLookupsEnabled();
+ private static int slowLookupThresholdMs = getSlowLookupThresholdMs();
+
/**
* For use only by tests and initialization
*/
@@ -480,9 +485,27 @@ public class SecurityUtil {
}
}
+ private static boolean getLogSlowLookupsEnabled() {
+ Configuration conf = new Configuration();
+
+ return conf.getBoolean(CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY,
+ CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT);
+ }
+
+ private static int getSlowLookupThresholdMs() {
+ Configuration conf = new Configuration();
+
+ return conf.getInt(CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY,
+ CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT);
+ }
+
/**
* Resolves a host subject to the security requirements determined by
- * hadoop.security.token.service.use_ip.
+ * hadoop.security.token.service.use_ip. Optionally logs slow resolutions.
*
* @param hostname host or ip to resolve
* @return a resolved host
@@ -491,7 +514,22 @@ public class SecurityUtil {
@InterfaceAudience.Private
public static
InetAddress getByName(String hostname) throws UnknownHostException {
- return hostResolver.getByName(hostname);
+ if (logSlowLookups || LOG.isTraceEnabled()) {
+ StopWatch lookupTimer = new StopWatch().start();
+ InetAddress result = hostResolver.getByName(hostname);
+ long elapsedMs = lookupTimer.stop().now(TimeUnit.MILLISECONDS);
+
+ if (elapsedMs >= slowLookupThresholdMs) {
+ LOG.warn("Slow name lookup for " + hostname + ". Took " + elapsedMs +
+ " ms.");
+ } else if (LOG.isTraceEnabled()) {
+ LOG.trace("Name lookup for " + hostname + " took " + elapsedMs +
+ " ms.");
+ }
+ return result;
+ } else {
+ return hostResolver.getByName(hostname);
+ }
}
interface HostResolver {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62180085/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b45f7bc..b582f15 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -122,6 +122,24 @@
</description>
</property>
+<property>
+ <name>hadoop.security.dns.log-slow-lookups.enabled</name>
+ <value>false</value>
+ <description>
+ Time name lookups (via SecurityUtil) and log them if they exceed the
+ configured threshold.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.dns.log-slow-lookups.threshold.ms</name>
+ <value>1000</value>
+ <description>
+ If slow lookup logging is enabled, this threshold is used to decide if a
+ lookup is considered slow enough to be logged.
+ </description>
+</property>
+
<!--
=== Multiple group mapping providers configuration sample ===
This sample illustrates a typical use case for CompositeGroupsMapping where