You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/08/16 03:58:21 UTC
[1/5] hadoop git commit: HDFS-10724. Document caller context config
keys. (Contributed by Mingliang Liu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 74156ee20 -> 2f4660ce9
HDFS-10724. Document caller context config keys. (Contributed by Mingliang Liu)
(cherry picked from commit 4bcbef39f7ca07601092919a7f2bea531a2dfa07)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e36a9136
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e36a9136
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e36a9136
Branch: refs/heads/branch-2
Commit: e36a913663ddb97469e76ddeb072d9530b6a1a29
Parents: 74156ee
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:20:33 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:36:11 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 29 ++++++++++++++++++++
1 file changed, 29 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e36a9136/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 7851632..77171ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2311,4 +2311,33 @@
needs to be specified in net.topology.script.file.name.
</description>
</property>
+
+ <property>
+ <name>hadoop.caller.context.enabled</name>
+ <value>false</value>
+ <description>When the feature is enabled, additional fields are written into
+ name-node audit log records for auditing coarse granularity operations.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.max.size</name>
+ <value>128</value>
+ <description>The maximum bytes a caller context string can have. If the
+ passed caller context is longer than this maximum bytes, client will
+ truncate it before sending to server. Note that the server may have a
+ different maximum size, and will truncate the caller context to the
+ maximum size it allows.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.signature.max.size</name>
+ <value>40</value>
+ <description>
+ The caller's signature (optional) is for offline validation. If the
+ signature exceeds the maximum allowed bytes in server, the caller context
+ will be abandoned, in which case the caller context will not be recorded
+ in audit logs.
+ </description>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/5] hadoop git commit: HDFS-10747. o.a.h.hdfs.tools.DebugAdmin
usage message is misleading. (Contributed by Mingliang Liu)
Posted by li...@apache.org.
HDFS-10747. o.a.h.hdfs.tools.DebugAdmin usage message is misleading. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82623ea9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82623ea9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82623ea9
Branch: refs/heads/branch-2
Commit: 82623ea9c9a95af64a23aca2bda61f4da65fde31
Parents: f9a7e59
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:23:47 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:38:17 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 4 ++--
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82623ea9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index d179a5c..a2b91ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -86,7 +86,7 @@ public class DebugAdmin extends Configured implements Tool {
private class VerifyBlockChecksumCommand extends DebugCommand {
VerifyBlockChecksumCommand() {
super("verify",
-"verify [-meta <metadata-file>] [-block <block-file>]",
+"verify -meta <metadata-file> [-block <block-file>]",
" Verify HDFS metadata and block files. If a block file is specified, we\n" +
" will verify that the checksums in the metadata file match the block\n" +
" file.");
@@ -200,7 +200,7 @@ public class DebugAdmin extends Configured implements Tool {
private class RecoverLeaseCommand extends DebugCommand {
RecoverLeaseCommand() {
super("recoverLease",
-"recoverLease [-path <path>] [-retries <num-retries>]",
+"recoverLease -path <path> [-retries <num-retries>]",
" Recover the lease on the specified path. The path must reside on an\n" +
" HDFS filesystem. The default number of retries is 1.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82623ea9/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index c224ba1..e54c113 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -556,7 +556,7 @@ Useful commands to help administrators debug HDFS issues, like validating block
### `verify`
-Usage: `hdfs debug verify [-meta <metadata-file>] [-block <block-file>]`
+Usage: `hdfs debug verify -meta <metadata-file> [-block <block-file>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -567,7 +567,7 @@ Verify HDFS metadata and block files. If a block file is specified, we will veri
### `recoverLease`
-Usage: `hdfs debug recoverLease [-path <path>] [-retries <num-retries>]`
+Usage: `hdfs debug recoverLease -path <path> [-retries <num-retries>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/5] hadoop git commit: HDFS-10678. Documenting
NNThroughputBenchmark tool. (Contributed by Mingliang Liu)
Posted by li...@apache.org.
HDFS-10678. Documenting NNThroughputBenchmark tool. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9a7e590
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9a7e590
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9a7e590
Branch: refs/heads/branch-2
Commit: f9a7e59066384c19b482231a1c1ed40a5324d829
Parents: e36a913
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:22:14 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:37:55 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/Benchmarking.md | 106 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 32 +-----
hadoop-project/src/site/site.xml | 1 +
3 files changed, 110 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a7e590/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md b/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
new file mode 100644
index 0000000..678dcee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
@@ -0,0 +1,106 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Hadoop Benchmarking
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+This page is to discuss benchmarking Hadoop using tools it provides.
+
+## NNThroughputBenchmark
+
+### Overview
+
+**NNThroughputBenchmark**, as its name indicates, is a name-node throughput benchmark, which runs a series of client threads on a single node against a name-node. If no name-node is configured, it will firstly start a name-node in the same process (_standalone mode_), in which case each client repetitively performs the same operation by directly calling the respective name-node methods. Otherwise, the benchmark will perform the operations against a remote name-node via client protocol RPCs (_remote mode_). Either way, all clients are running locally in a single process rather than remotely across different nodes. The reason is to avoid communication overhead caused by RPC connections and serialization, and thus reveal the upper bound of pure name-node performance.
+
+The benchmark first generates inputs for each thread so that the input generation overhead does not effect the resulting statistics. The number of operations performed by threads is practically the same. Precisely, the difference between the number of operations performed by any two threads does not exceed 1. Then the benchmark executes the specified number of operations using the specified number of threads and outputs the resulting stats by measuring the number of operations performed by the name-node per second.
+
+### Commands
+
+The general command line syntax is:
+
+`hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark [genericOptions] [commandOptions]`
+
+#### Generic Options
+
+This benchmark honors the [Hadoop command-line Generic Options](CommandsManual.html#Generic_Options) to alter its behavior. The benchmark, as other tools, will rely on the `fs.defaultFS` config, which is overridable by `-fs` command option, to run standalone mode or remote mode. If the `fs.defaultFS` scheme is not specified or is `file` (local), the benchmark will run in _standalone mode_. Specially, the _remote_ name-node config `dfs.namenode.fs-limits.min-block-size` should be set as 16 while in _standalone mode_ the benchmark turns off minimum block size verification for its internal name-node.
+
+#### Command Options
+
+The following are all supported command options:
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+|`-op` | Specify the operation. This option must be provided and should be the first option. |
+|`-logLevel` | Specify the logging level when the benchmark runs. The default logging level is ERROR. |
+|`-UGCacheRefreshCount` | After every specified number of operations, the benchmark purges the name-node's user group cache. By default the refresh is never called. |
+|`-keepResults` | If specified, do not clean up the name-space after execution. By default the name-space will be removed after test. |
+
+##### Operations Supported
+
+Following are all the operations supported along with their respective operation-specific parameters (all optional) and default values.
+
+| OPERATION\_OPTION | Operation-specific parameters |
+|:---- |:---- |
+|`all` | _options for other operations_ |
+|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] |
+|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] |
+|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] |
+|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] |
+|`clean` | N/A |
+
+##### Operation Options
+
+When running benchmarks with the above operation(s), please provide operation-specific parameters illustrated as following.
+
+| OPERATION\_SPECIFIC\_OPTION | Description |
+|:---- |:---- |
+|`-threads` | Number of total threads to run the respective operation. |
+|`-files` | Number of total files for the respective operation. |
+|`-dirs` | Number of total directories for the respective operation. |
+|`-filesPerDir` | Number of files per directory. |
+|`-close` | Close the files after creation. |
+|`-dirsPerDir` | Number of directories per directory. |
+|`-useExisting` | If specified, do not recreate the name-space, use existing data. |
+|`-datanodes` | Total number of simulated data-nodes. |
+|`-reports` | Total number of block reports to send. |
+|`-blocksPerReport` | Number of blocks per report. |
+|`-blocksPerFile` | Number of blocks per file. |
+|`-nodesToDecommission` | Total number of simulated data-nodes to decommission. |
+|`-nodeReplicationLimit` | The maximum number of outgoing replication streams for a data-node. |
+|`-totalBlocks` | Number of total blocks to operate. |
+|`-replication` | Replication factor. Will be adjusted to number of data-nodes if it is larger than that. |
+
+### Reports
+
+The benchmark measures the number of operations performed by the name-node per second. Specifically, for each operation tested, it reports the total running time in seconds (_Elapsed Time_), operation throughput (_Ops per sec_), and average time for the operations (_Average Time_). The higher, the better.
+
+Following is a sample reports by running following commands that opens 100K files with 1K threads against a remote name-node. See [HDFS scalability: the limits to growth](https://www.usenix.org/legacy/publications/login/2010-04/openpdfs/shvachko.pdf) for real-world benchmark stats.
+```
+$ hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark -fs hdfs://nameservice:9000 -op open -threads 1000 -files 100000
+
+--- open inputs ---
+nrFiles = 100000
+nrThreads = 1000
+nrFilesPerDir = 4
+--- open stats ---
+# operations: 100000
+Elapsed Time: 9510
+ Ops per sec: 10515.247108307045
+Average Time: 90
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a7e590/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 6744571..f569a3a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -95,36 +95,10 @@ import org.apache.log4j.LogManager;
* except for the name-node. Each operation is executed
* by calling directly the respective name-node method.
* The name-node here is real all other components are simulated.
- *
- * This benchmark supports
- * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
- * standard command-line options</a>. If you use remote namenode by <tt>-fs</tt>
- * option, its config <tt>dfs.namenode.fs-limits.min-block-size</tt> should be
- * set as 16.
*
- * Command line arguments for the benchmark include:
- * <ol>
- * <li>total number of operations to be performed,</li>
- * <li>number of threads to run these operations,</li>
- * <li>followed by operation specific input parameters.</li>
- * <li>-logLevel L specifies the logging level when the benchmark runs.
- * The default logging level is {@link Level#ERROR}.</li>
- * <li>-UGCacheRefreshCount G will cause the benchmark to call
- * {@link NameNodeRpcServer#refreshUserToGroupsMappings} after
- * every G operations, which purges the name-node's user group cache.
- * By default the refresh is never called.</li>
- * <li>-keepResults do not clean up the name-space after execution.</li>
- * <li>-useExisting do not recreate the name-space, use existing data.</li>
- * </ol>
- *
- * The benchmark first generates inputs for each thread so that the
- * input generation overhead does not effect the resulting statistics.
- * The number of operations performed by threads is practically the same.
- * Precisely, the difference between the number of operations
- * performed by any two threads does not exceed 1.
- *
- * Then the benchmark executes the specified number of operations using
- * the specified number of threads and outputs the resulting stats.
+ * For usage, please see <a href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Benchmarking.html#NNThroughputBenchmark">the documentation</a>.
+ * Meanwhile, if you change the usage of this program, please also update the
+ * documentation accordingly.
*/
public class NNThroughputBenchmark implements Tool {
private static final Log LOG = LogFactory.getLog(NNThroughputBenchmark.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a7e590/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 0167f0c..0705cae 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -162,6 +162,7 @@
<item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
+ <item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>
<menu name="Reference" inherit="top">
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/5] hadoop git commit: HDFS-10641.
TestBlockManager#testBlockReportQueueing fails intermittently. (Contributed
by Daryn Sharp)
Posted by li...@apache.org.
HDFS-10641. TestBlockManager#testBlockReportQueueing fails intermittently. (Contributed by Daryn Sharp)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f4660ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f4660ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f4660ce
Branch: refs/heads/branch-2
Commit: 2f4660ce9892e822d241c87954219460e4351779
Parents: 23161c6
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:28:40 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:39:12 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/TestBlockManager.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f4660ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 5d0ca96..cc0fafd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -945,6 +945,7 @@ public class TestBlockManager {
final CyclicBarrier startBarrier = new CyclicBarrier(2);
final CountDownLatch endLatch = new CountDownLatch(3);
+ final CountDownLatch doneLatch = new CountDownLatch(1);
// create a task intended to block while processing, thus causing
// the queue to backup. simulates how a full BR is processed.
@@ -952,7 +953,7 @@ public class TestBlockManager {
new Callable<Void>(){
@Override
public Void call() throws IOException {
- return bm.runBlockOp(new Callable<Void>() {
+ bm.runBlockOp(new Callable<Void>() {
@Override
public Void call()
throws InterruptedException, BrokenBarrierException {
@@ -962,6 +963,9 @@ public class TestBlockManager {
return null;
}
});
+ // signal that runBlockOp returned
+ doneLatch.countDown();
+ return null;
}
});
@@ -1006,7 +1010,7 @@ public class TestBlockManager {
startBarrier.await(1, TimeUnit.SECONDS);
assertTrue(endLatch.await(1, TimeUnit.SECONDS));
assertEquals(0, bm.getBlockOpQueueLength());
- assertTrue(blockingOp.isDone());
+ assertTrue(doneLatch.await(1, TimeUnit.SECONDS));
} finally {
cluster.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/5] hadoop git commit: HADOOP-13470. GenericTestUtils$LogCapturer
is flaky. (Contributed by Mingliang Liu)
Posted by li...@apache.org.
HADOOP-13470. GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23161c67
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23161c67
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23161c67
Branch: refs/heads/branch-2
Commit: 23161c67cf1420de417461614d4af86fefe24f50
Parents: 82623ea
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:24:54 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:38:40 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/test/GenericTestUtils.java | 25 ++++++-----
.../hadoop/test/TestGenericTestUtils.java | 44 ++++++++++++++++++++
2 files changed, 56 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23161c67/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 116a111..6b5135c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -42,10 +42,10 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
-import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
import org.junit.Assume;
@@ -275,36 +275,35 @@ public abstract class GenericTestUtils {
private StringWriter sw = new StringWriter();
private WriterAppender appender;
private Logger logger;
-
+
public static LogCapturer captureLogs(Log l) {
Logger logger = ((Log4JLogger)l).getLogger();
- LogCapturer c = new LogCapturer(logger);
- return c;
+ return new LogCapturer(logger);
+ }
+
+ public static LogCapturer captureLogs(org.slf4j.Logger logger) {
+ return new LogCapturer(toLog4j(logger));
}
-
private LogCapturer(Logger logger) {
this.logger = logger;
- Layout layout = Logger.getRootLogger().getAppender("stdout").getLayout();
- WriterAppender wa = new WriterAppender(layout, sw);
- logger.addAppender(wa);
+ this.appender = new WriterAppender(new PatternLayout(), sw);
+ logger.addAppender(appender);
}
-
+
public String getOutput() {
return sw.toString();
}
-
+
public void stopCapturing() {
logger.removeAppender(appender);
-
}
public void clearOutput() {
sw.getBuffer().setLength(0);
}
}
-
-
+
/**
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23161c67/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
index 8a7b5f6..86df5d5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
@@ -18,8 +18,16 @@
package org.apache.hadoop.test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
public class TestGenericTestUtils extends GenericTestUtils {
@Test
@@ -75,4 +83,40 @@ public class TestGenericTestUtils extends GenericTestUtils {
}
}
+ @Test(timeout = 10000)
+ public void testLogCapturer() {
+ final Log log = LogFactory.getLog(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(log);
+ final String infoMessage = "info message";
+ // test get output message
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
+ @Test(timeout = 10000)
+ public void testLogCapturerSlf4jLogger() {
+ final Logger logger = LoggerFactory.getLogger(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(logger);
+ final String infoMessage = "info message";
+ // test get output message
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org