You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/08/28 00:23:41 UTC
[14/22] git commit: HDFS-6773. MiniDFSCluster should skip edit log
fsync by default. Contributed by Stephen Chu.
HDFS-6773. MiniDFSCluster should skip edit log fsync by default. Contributed by Stephen Chu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d805cc27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d805cc27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d805cc27
Branch: refs/heads/HDFS-6581
Commit: d805cc27a98abbdf14a20ef3127a2c7cb212c765
Parents: fdd3bc5
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Aug 27 13:33:02 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Aug 27 13:33:02 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 +++++++++++++++++---
.../server/datanode/TestFsDatasetCache.java | 1 -
.../server/namenode/TestCacheDirectives.java | 1 -
4 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 99d5c01..7783243 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6690. Deduplicate xattr names in memory. (wang)
+ HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen
+ Chu via Colin Patrick McCabe)
+
BUG FIXES
HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 98ca316..0e49cfe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -93,6 +93,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -172,6 +173,7 @@ public class MiniDFSCluster {
private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false;
private Configuration[] dnConfOverlays;
+ private boolean skipFsyncForTesting = true;
public Builder(Configuration conf) {
this.conf = conf;
@@ -405,6 +407,15 @@ public class MiniDFSCluster {
this.dnConfOverlays = dnConfOverlays;
return this;
}
+
+ /**
+ * Default: true
+ * When true, we skip fsync() calls for speed improvements.
+ */
+ public Builder skipFsyncForTesting(boolean val) {
+ this.skipFsyncForTesting = val;
+ return this;
+ }
/**
* Construct the actual MiniDFSCluster
@@ -472,7 +483,8 @@ public class MiniDFSCluster {
builder.checkExitOnShutdown,
builder.checkDataNodeAddrConfig,
builder.checkDataNodeHostConfig,
- builder.dnConfOverlays);
+ builder.dnConfOverlays,
+ builder.skipFsyncForTesting);
}
public class DataNodeProperties {
@@ -727,7 +739,8 @@ public class MiniDFSCluster {
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, null, racks, hosts,
null, simulatedCapacities, null, true, false,
- MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null);
+ MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
+ true, false, false, null, true);
}
private void initMiniDFSCluster(
@@ -742,7 +755,8 @@ public class MiniDFSCluster {
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig,
- Configuration[] dnConfOverlays)
+ Configuration[] dnConfOverlays,
+ boolean skipFsyncForTesting)
throws IOException {
boolean success = false;
try {
@@ -782,6 +796,8 @@ public class MiniDFSCluster {
+ "Standby node since no IPC ports have been specified.");
conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
}
+
+ EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting);
federation = nnTopology.isFederated();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
index 5ac13ee..d6e70d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
@@ -108,7 +108,6 @@ public class TestFsDatasetCache {
private static CacheManipulator prevCacheManipulator;
static {
- EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index d54b90e..9307692 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -110,7 +110,6 @@ public class TestCacheDirectives {
static {
NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
- EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
}
private static final long BLOCK_SIZE = 4096;