You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2017/01/16 22:43:56 UTC
hadoop git commit: HDFS-11339. Support File IO sampling for Datanode
IO profiling hooks. Contributed by Hanisha Koneru.
Repository: hadoop
Updated Branches:
refs/heads/trunk a853b4e1b -> 79e939d0b
HDFS-11339. Support File IO sampling for Datanode IO profiling hooks. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79e939d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79e939d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79e939d0
Branch: refs/heads/trunk
Commit: 79e939d0b848a50200612c8c471db6bce1c822be
Parents: a853b4e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jan 16 14:43:29 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jan 16 14:43:29 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +++++++++++----
.../server/datanode/ProfilingFileIoEvents.java | 21 ++++++++++++++++++--
.../hadoop/tools/TestHdfsConfigFields.java | 2 ++
3 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index a00e872..8f60af0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -694,10 +694,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize";
public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024;
- //Keys with no defaults
- public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
- public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
- public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy";
+ // Datanode File IO Stats
public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
"dfs.datanode.enable.fileio.profiling";
public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
@@ -706,6 +703,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_DEFAULT = false;
+ public static final String
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
+ "dfs.datanode.fileio.profiling.sampling.fraction";
+ public static final double
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
+
+ //Keys with no defaults
+ public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
+ public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
+ public static final String DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY = "dfs.datanode.fsdataset.volume.choosing.policy";
+
public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold";
public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB
public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 43ac495..6e689a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hdfs.server.datanode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.util.Time;
+import org.jboss.netty.util.internal.ThreadLocalRandom;
import javax.annotation.Nullable;
@@ -33,16 +36,30 @@ import javax.annotation.Nullable;
*/
@InterfaceAudience.Private
class ProfilingFileIoEvents {
+ static final Log LOG = LogFactory.getLog(ProfilingFileIoEvents.class);
private final boolean isEnabled;
+ private final int sampleRangeMax;
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
isEnabled = conf.getBoolean(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
+ double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
+ DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+ if (fileIOSamplingFraction > 1) {
+ LOG.warn(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
+ " value cannot be more than 1. Setting value to 1");
+ fileIOSamplingFraction = 1;
+ }
+ sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
isEnabled = false;
+ sampleRangeMax = 0;
}
}
@@ -69,7 +86,7 @@ class ProfilingFileIoEvents {
public long beforeFileIo(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op, long len) {
- if (isEnabled) {
+ if (isEnabled && ThreadLocalRandom.current().nextInt() < sampleRangeMax) {
DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
if (metrics != null) {
return Time.monotonicNow();
@@ -80,7 +97,7 @@ class ProfilingFileIoEvents {
public void afterFileIo(@Nullable FsVolumeSpi volume,
FileIoProvider.OPERATION op, long begin, long len) {
- if (isEnabled) {
+ if (isEnabled && begin != 0) {
DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
if (metrics != null) {
long latency = Time.monotonicNow() - begin;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e939d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 51ef5d0..6b0dced 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
+ configurationPropsToSkipCompare.add(DFSConfigKeys
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY);
// Allocate
xmlPropsToSkipCompare = new HashSet<String>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org