You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by kk...@apache.org on 2017/03/07 23:10:28 UTC
[14/50] hadoop git commit: Revert "HDFS-11477. Simplify file IO
profiling configuration. Contributed by Hanisha Koneru."
Revert "HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru."
This reverts commit e61491d4769536ce8b77b416005ce531278ce729.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f4817f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f4817f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f4817f2
Branch: refs/heads/YARN-5972
Commit: 8f4817f2c5524c1891a3b3b50e396dd4ca294a2d
Parents: e61491d
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Mar 2 16:20:48 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Mar 2 16:20:48 2017 -0800
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 7 +----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++-
.../hdfs/server/datanode/FileIoProvider.java | 2 +-
.../server/datanode/ProfilingFileIoEvents.java | 27 +++++---------------
.../datanode/TestDataNodeVolumeMetrics.java | 4 +--
.../hadoop/tools/TestHdfsConfigFields.java | 2 ++
6 files changed, 17 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index a8bdbeb..7900692 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -332,12 +332,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
FsVolume
--------
-Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
-metrics are off by default. They can be enabled by setting `dfs.datanode
-.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
-Setting this value to 0.0 would mean profiling is not enabled. But enabling
-per-volume metrics may have a performance impact. Each metrics record
-contains tags such as Hostname as additional information along with metrics.
+Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
| Name | Description |
|:---- |:---- |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 82d6073..be20829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -718,6 +718,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
// Datanode File IO Stats
+ public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
+ "dfs.datanode.enable.fileio.profiling";
+ public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
+ false;
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
"dfs.datanode.enable.fileio.fault.injection";
public static final boolean
@@ -726,7 +730,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
"dfs.datanode.fileio.profiling.sampling.fraction";
public static final double
- DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
+ DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 5508e0b..9def2e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
*
* Behavior can be injected into these events by enabling the
* profiling and/or fault injection event hooks through
- * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
+ * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
* These event hooks are disabled by default.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 7eab969..9af84fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.datanode;
-import static java.lang.Math.abs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,32 +43,18 @@ class ProfilingFileIoEvents {
public ProfilingFileIoEvents(@Nullable Configuration conf) {
if (conf != null) {
+ isEnabled = conf.getBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
- if (abs(fileIOSamplingFraction) < 0.000001) {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Disabling file IO profiling");
- isEnabled = false;
- } else if (fileIOSamplingFraction < 0.000001) {
+ .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
+ if (fileIOSamplingFraction > 1) {
LOG.warn(DFSConfigKeys
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be less than 0. Disabling file IO profiling.");
- isEnabled = false;
- } else if (fileIOSamplingFraction > 1) {
- LOG.warn(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
- " value cannot be more than 1. Setting value to 1 and enabling " +
- "file IO profiling");
- isEnabled = true;
+ " value cannot be more than 1. Setting value to 1");
fileIOSamplingFraction = 1;
- } else {
- LOG.info(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
- + fileIOSamplingFraction + ". Enabling file IO profiling");
- isEnabled = true;
}
sampleRangeMax = (int) (fileIOSamplingFraction * Integer.MAX_VALUE);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 03e1dee..6a8ac9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setDouble(DFSConfigKeys
- .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+ conf.setBoolean(DFSConfigKeys
+ .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
SimulatedFSDataset.setFactory(conf);
return new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4817f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 1fdf713..cdce342 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -103,6 +103,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
configurationPropsToSkipCompare
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
configurationPropsToSkipCompare.add(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org