You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/06/21 23:24:16 UTC
[29/50] [abbrv] hadoop git commit: HDFS-11345. Document the
configuration key for FSNamesystem lock fairness. Contributed by Erik Krogen.
HDFS-11345. Document the configuration key for FSNamesystem lock fairness. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c769167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c769167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c769167
Branch: refs/heads/HDFS-10467
Commit: 2c769167dbdb66c52d2ba7b7193a686444085570
Parents: 15964ef
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Jun 20 16:35:54 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Jun 20 16:35:54 2017 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++
.../hadoop/hdfs/server/namenode/FSNamesystemLock.java | 5 ++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++
.../hadoop/hdfs/server/namenode/TestFSNamesystemLock.java | 7 ++++---
4 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c769167/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index eaaff60..e7aba4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -423,6 +423,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long
DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25;
+ public static final String DFS_NAMENODE_FSLOCK_FAIR_KEY =
+ "dfs.namenode.fslock.fair";
+ public static final boolean DFS_NAMENODE_FSLOCK_FAIR_DEFAULT = true;
+
public static final String DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY =
"dfs.namenode.lock.detailed-metrics.enabled";
public static final boolean DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c769167/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index be33c4b..8c60faa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.util.Timer;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
@@ -113,7 +115,8 @@ class FSNamesystemLock {
@VisibleForTesting
FSNamesystemLock(Configuration conf,
MutableRatesWithAggregation detailedHoldTimeMetrics, Timer timer) {
- boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
+ boolean fair = conf.getBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY,
+ DFS_NAMENODE_FSLOCK_FAIR_DEFAULT);
FSNamesystem.LOG.info("fsLock is fair: " + fair);
this.coarseLock = new ReentrantReadWriteLock(fair);
this.timer = timer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c769167/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index dbf78fc..0f72041 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2760,6 +2760,16 @@
</property>
<property>
+ <name>dfs.namenode.fslock.fair</name>
+ <value>true</value>
+ <description>If this is true, the FS Namesystem lock will be used in Fair mode,
+ which will help to prevent writer threads from being starved, but can provide
+ lower lock throughput. See java.util.concurrent.locks.ReentrantReadWriteLock
+ for more information on fair/non-fair locks.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>0</value>
<description>The delay in seconds at which we will pause the blocks deletion
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c769167/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
index 1af8bf7..7f9746d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
@@ -40,6 +40,7 @@ import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import static org.junit.Assert.*;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
@@ -53,11 +54,11 @@ public class TestFSNamesystemLock {
public void testFsLockFairness() throws IOException, InterruptedException{
Configuration conf = new Configuration();
- conf.setBoolean("dfs.namenode.fslock.fair", true);
+ conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null);
assertTrue(fsnLock.coarseLock.isFair());
- conf.setBoolean("dfs.namenode.fslock.fair", false);
+ conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, false);
fsnLock = new FSNamesystemLock(conf, null);
assertFalse(fsnLock.coarseLock.isFair());
}
@@ -103,7 +104,7 @@ public class TestFSNamesystemLock {
final int threadCount = 3;
final CountDownLatch latch = new CountDownLatch(threadCount);
final Configuration conf = new Configuration();
- conf.setBoolean("dfs.namenode.fslock.fair", true);
+ conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
final FSNamesystemLock rwLock = new FSNamesystemLock(conf, null);
rwLock.writeLock();
ExecutorService helper = Executors.newFixedThreadPool(threadCount);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org