You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2022/04/19 03:14:09 UTC
[hadoop] branch branch-3.3 updated: HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new cfe2d8aa79d HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819)
cfe2d8aa79d is described below
commit cfe2d8aa79d5a2d0c9915019155ac5df97c7f9c2
Author: jianghuazhu <74...@qq.com>
AuthorDate: Mon Apr 18 11:21:51 2022 +0800
HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819)
Reviewed-by: Viraj Jasani <vj...@apache.org>
Reviewed-by: Akira Ajisaka <aa...@apache.org>
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit 900682e7120e7af250ebf0b4bafb40b412d3e11c)
---
.../hdfs/server/namenode/FileNameGenerator.java | 4 +++
.../server/namenode/NNThroughputBenchmark.java | 40 +++++++++++++++-------
2 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
index 6f7087b9215..ed18696c232 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
@@ -77,6 +77,10 @@ public class FileNameGenerator {
return fn;
}
+ public long getFileCount() {
+ return fileCount;
+ }
+
private synchronized void reset() {
Arrays.fill(pathIndecies, -1);
fileCount = 0L;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 614a43e70a9..9537435a59c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -570,12 +570,20 @@ public class NNThroughputBenchmark implements Tool {
// int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
fileNames = new String[numThreads][];
- for(int idx=0; idx < numThreads; idx++) {
- int threadOps = opsPerThread[idx];
- fileNames[idx] = new String[threadOps];
- for(int jdx=0; jdx < threadOps; jdx++)
- fileNames[idx][jdx] = nameGenerator.
- getNextFileName("ThroughputBench");
+ try {
+ for(int idx=0; idx < numThreads; idx++) {
+ int threadOps = opsPerThread[idx];
+ fileNames[idx] = new String[threadOps];
+ for(int jdx=0; jdx < threadOps; jdx++) {
+ fileNames[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
+ }
+ }
+ } catch (ArrayIndexOutOfBoundsException e) {
+ LOG.error("The current environment allows {} files to be created. " +
+ "If you want to test more files, please update the -filesPerDir parameter.",
+ nameGenerator.getFileCount());
+ throw e;
}
}
@@ -670,12 +678,20 @@ public class NNThroughputBenchmark implements Tool {
false);
LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
dirPaths = new String[numThreads][];
- for(int idx=0; idx < numThreads; idx++) {
- int threadOps = opsPerThread[idx];
- dirPaths[idx] = new String[threadOps];
- for(int jdx=0; jdx < threadOps; jdx++)
- dirPaths[idx][jdx] = nameGenerator.
- getNextFileName("ThroughputBench");
+ try {
+ for(int idx=0; idx < numThreads; idx++) {
+ int threadOps = opsPerThread[idx];
+ dirPaths[idx] = new String[threadOps];
+ for(int jdx=0; jdx < threadOps; jdx++) {
+ dirPaths[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
+ }
+ }
+ } catch (ArrayIndexOutOfBoundsException e) {
+ LOG.error("The current environment allows {} directories to be created. " +
+ "If you want to test more directories, please update the -dirsPerDir parameter.",
+ nameGenerator.getFileCount());
+ throw e;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org