You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/07/03 19:53:11 UTC
[3/3] hadoop git commit: HDFS-9503. Use generic option -fs for
NNThroughputBenchmark instead of -namenode. Contributed by Mingliang Liu.
HDFS-9503. Use generic option -fs for NNThroughputBenchmark instead of -namenode. Contributed by Mingliang Liu.
(cherry picked from commit 81ef1a1f55ee0104f267316e44596463a78f87ea)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53907b66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53907b66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53907b66
Branch: refs/heads/branch-2.7
Commit: 53907b66cb1b6255ff9f9e36af095662287dbdd2
Parents: 2af5548
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Tue Feb 2 14:52:53 2016 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Jul 3 12:23:00 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/namenode/NNThroughputBenchmark.java | 59 +++++++---------
.../namenode/TestNNThroughputBenchmark.java | 72 ++++++++++++++++++--
3 files changed, 95 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53907b66/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 953e63a..716167c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -111,6 +111,9 @@ Release 2.7.4 - UNRELEASED
HDFS-9421. NNThroughputBenchmark replication test NPE with -namenode option.
(Mingliang Liu via xyao)
+ HDFS-9503. Use generic option -fs for NNThroughputBenchmark instead of
+ -namenode. (Mingliang Liu via shv)
+
OPTIMIZATIONS
HDFS-10896. Move lock logging logic from FSNamesystem into FSNamesystemLock.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53907b66/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 72e0eef..ff7cfae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -74,6 +74,8 @@ import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
@@ -93,6 +95,12 @@ import org.apache.log4j.LogManager;
* by calling directly the respective name-node method.
* The name-node here is real all other components are simulated.
*
+ * This benchmark supports
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ * standard command-line options</a>. If you use remote namenode by <tt>-fs</tt>
+ * option, its config <tt>dfs.namenode.fs-limits.min-block-size</tt> should be
+ * set as 16.
+ *
* Command line arguments for the benchmark include:
* <ol>
* <li>total number of operations to be performed,</li>
@@ -106,9 +114,6 @@ import org.apache.log4j.LogManager;
* By default the refresh is never called.</li>
* <li>-keepResults do not clean up the name-space after execution.</li>
* <li>-useExisting do not recreate the name-space, use existing data.</li>
- * <li>-namenode will run the test (except {@link ReplicationStats}) against a
- * namenode in another process or on another host. If you use this option,
- * the namenode must have dfs.namenode.fs-limits.min-block-size set to 16.</li>
* </ol>
*
* The benchmark first generates inputs for each thread so that the
@@ -123,12 +128,8 @@ import org.apache.log4j.LogManager;
public class NNThroughputBenchmark implements Tool {
private static final Log LOG = LogFactory.getLog(NNThroughputBenchmark.class);
private static final int BLOCK_SIZE = 16;
- private static final String GENERAL_OPTIONS_USAGE =
- " [-keepResults] | [-logLevel L] | [-UGCacheRefreshCount G] |" +
- " [-namenode <namenode URI>]\n" +
- " If using -namenode, set the namenode's " +
- "dfs.namenode.fs-limits.min-block-size to 16. Replication test does not " +
- "support -namenode.";
+ private static final String GENERAL_OPTIONS_USAGE =
+ "[-keepResults] | [-logLevel L] | [-UGCacheRefreshCount G]";
static Configuration config;
static NameNode nameNode;
@@ -138,8 +139,6 @@ public class NNThroughputBenchmark implements Tool {
static RefreshUserMappingsProtocol refreshUserMappingsProto;
static String bpid = null;
- private String namenodeUri = null; // NN URI to use, if specified
-
NNThroughputBenchmark(Configuration conf) throws IOException {
config = conf;
// We do not need many handlers, since each thread simulates a handler
@@ -383,12 +382,6 @@ public class NNThroughputBenchmark implements Tool {
args.remove(ugrcIndex);
}
- try {
- namenodeUri = StringUtils.popOptionWithArgument("-namenode", args);
- } catch (IllegalArgumentException iae) {
- printUsage();
- }
-
String type = args.get(1);
if(OP_ALL_NAME.equals(type)) {
type = getOpName();
@@ -1442,15 +1435,19 @@ public class NNThroughputBenchmark implements Tool {
+ " | \n\t" + CleanAllStats.OP_CLEAN_USAGE
+ " | \n\t" + GENERAL_OPTIONS_USAGE
);
- System.exit(-1);
+ System.err.println();
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ System.err.println("If connecting to a remote NameNode with -fs option, " +
+ "dfs.namenode.fs-limits.min-block-size should be set to 16.");
+ ExitUtil.terminate(-1);
}
- public static void runBenchmark(Configuration conf, List<String> args)
+ public static void runBenchmark(Configuration conf, String[] args)
throws Exception {
NNThroughputBenchmark bench = null;
try {
bench = new NNThroughputBenchmark(conf);
- bench.run(args.toArray(new String[]{}));
+ ToolRunner.run(bench, args);
} finally {
if(bench != null)
bench.close();
@@ -1470,6 +1467,7 @@ public class NNThroughputBenchmark implements Tool {
String type = args.get(1);
boolean runAll = OperationStatsBase.OP_ALL_NAME.equals(type);
+ final URI nnUri = FileSystem.getDefaultUri(config);
// Start the NameNode
String[] argv = new String[] {};
@@ -1505,10 +1503,9 @@ public class NNThroughputBenchmark implements Tool {
ops.add(opStat);
}
if(runAll || ReplicationStats.OP_REPLICATION_NAME.equals(type)) {
- if (namenodeUri != null || args.contains("-namenode")) {
+ if (nnUri.getScheme() != null && nnUri.getScheme().equals("hdfs")) {
LOG.warn("The replication test is ignored as it does not support " +
- "standalone namenode in another process or on another host. " +
- "Please run replication test without -namenode argument.");
+ "standalone namenode in another process or on another host. ");
} else {
opStat = new ReplicationStats(args);
ops.add(opStat);
@@ -1522,7 +1519,10 @@ public class NNThroughputBenchmark implements Tool {
printUsage();
}
- if (namenodeUri == null) {
+ if (nnUri.getScheme() == null || nnUri.getScheme().equals("file")) {
+ LOG.info("Remote NameNode is not specified. Creating one.");
+ FileSystem.setDefaultUri(config, "hdfs://localhost:0");
+ config.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
nameNode = NameNode.createNameNode(argv, config);
NamenodeProtocols nnProtos = nameNode.getRpcServer();
nameNodeProto = nnProtos;
@@ -1531,10 +1531,8 @@ public class NNThroughputBenchmark implements Tool {
refreshUserMappingsProto = nnProtos;
bpid = nameNode.getNamesystem().getBlockPoolId();
} else {
- FileSystem.setDefaultUri(getConf(), namenodeUri);
DistributedFileSystem dfs = (DistributedFileSystem)
FileSystem.get(getConf());
- final URI nnUri = new URI(namenodeUri);
nameNodeProto = DFSTestUtil.getNamenodeProtocolProxy(config, nnUri,
UserGroupInformation.getCurrentUser());
clientProto = dfs.getClient().getNamenode();
@@ -1569,14 +1567,7 @@ public class NNThroughputBenchmark implements Tool {
}
public static void main(String[] args) throws Exception {
- NNThroughputBenchmark bench = null;
- try {
- bench = new NNThroughputBenchmark(new HdfsConfiguration());
- ToolRunner.run(bench, args);
- } finally {
- if(bench != null)
- bench.close();
- }
+ runBenchmark(new HdfsConfiguration(), args);
}
@Override // Configurable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53907b66/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
index d964230..9f1ebd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
-import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -27,11 +26,18 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.util.ExitUtil;
import org.junit.After;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestNNThroughputBenchmark {
+ @BeforeClass
+ public static void setUp() {
+ ExitUtil.disableSystemExit();
+ }
+
@After
public void cleanUp() {
FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory()));
@@ -46,10 +52,66 @@ public class TestNNThroughputBenchmark {
File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
nameDir.getAbsolutePath());
- FileSystem.setDefaultUri(conf, "hdfs://localhost:" + 0);
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
DFSTestUtil.formatNameNode(conf);
- String[] args = new String[] {"-op", "all"};
- NNThroughputBenchmark.runBenchmark(conf, Arrays.asList(args));
+ NNThroughputBenchmark.runBenchmark(conf, new String[] {"-op", "all"});
+ }
+
+ /**
+ * This test runs all benchmarks defined in {@link NNThroughputBenchmark},
+ * with explicit local -fs option.
+ */
+ @Test(timeout = 120000)
+ public void testNNThroughputWithFsOption() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
+ DFSTestUtil.formatNameNode(conf);
+ NNThroughputBenchmark.runBenchmark(conf,
+ new String[] {"-fs", "file:///", "-op", "all"});
+ }
+
+ /**
+ * This test runs {@link NNThroughputBenchmark} against a mini DFS cluster.
+ */
+ @Test(timeout = 120000)
+ public void testNNThroughputAgainstRemoteNN() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+
+ final Configuration benchConf = new HdfsConfiguration();
+ FileSystem.setDefaultUri(benchConf, cluster.getURI());
+ NNThroughputBenchmark.runBenchmark(benchConf, new String[]{"-op", "all"});
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
+ * with explicit -fs option.
+ */
+ @Test(timeout = 120000)
+ public void testNNThroughputRemoteAgainstNNWithFsOption() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+
+ NNThroughputBenchmark.runBenchmark(new HdfsConfiguration(),
+ new String[]{"-fs", cluster.getURI().toString(), "-op", "all"});
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org