You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2013/10/24 23:38:28 UTC
svn commit: r1535563 - in
/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/ src/ src/main/bin/ src/main/java/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ src/main/j...
Author: wang
Date: Thu Oct 24 21:38:25 2013
New Revision: 1535563
URL: http://svn.apache.org/r1535563
Log:
merge trunk into HDFS-4949 branch
Added:
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfs-dust.js
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfs-dust.js
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.dust.html
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.dust.html
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer-block-info.dust.html
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer-block-info.dust.html
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.dust.html
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.dust.html
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestURLConnectionFactory.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenRemoteFetcher.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenRemoteFetcher.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
Modified:
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/posix_util.c
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1532946-1535559
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Oct 24 21:38:25 2013
@@ -255,7 +255,15 @@ Release 2.3.0 - UNRELEASED
HDFS-5342. Provide more information in the FSNamesystem JMX interfaces.
(Haohui Mai via jing9)
-
+
+ HDFS-5334. Implement dfshealth.jsp in HTML pages. (Haohui Mai via jing9)
+
+ HDFS-5379. Update links to datanode information in dfshealth.html. (Haohui
+ Mai via jing9)
+
+ HDFS-5382. Implement the UI of browsing filesystems in HTML 5 page. (Haohui
+ Mai via jing9)
+
IMPROVEMENTS
HDFS-5267. Remove volatile from LightWeightHashSet. (Junping Du via llu)
@@ -319,10 +327,24 @@ Release 2.3.0 - UNRELEASED
HDFS-5130. Add test for snapshot related FsShell and DFSAdmin commands.
(Binglin Chang via jing9)
+ HDFS-5374. Remove deadcode in DFSOutputStream. (suresh)
+
+ HDFS-4511. Cover package org.apache.hadoop.hdfs.tools with unit test
+ (Andrey Klochkov via jeagles)
+
+ HDFS-4885. Improve the verifyBlockPlacement() API in BlockPlacementPolicy.
+ (Junping Du via szetszwo)
+
+ HDFS-5363. Refactor WebHdfsFileSystem: move SPENGO-authenticated connection
+ creation to URLConnectionFactory. (Haohui Mai via jing9)
+
OPTIMIZATIONS
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
+ HDFS-5341. Reduce fsdataset lock duration during directory scanning.
+ (Qus-Jiawei via kihwal)
+
BUG FIXES
HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin
Patrick McCabe)
@@ -354,6 +376,14 @@ Release 2.3.0 - UNRELEASED
HDFS-4376. Fix race conditions in Balancer. (Junping Du via szetszwo)
+ HDFS-5375. hdfs.cmd does not expose several snapshot commands. (cnauroth)
+
+ HDFS-5336. DataNode should not output 'StartupProgress' metrics.
+ (Akira Ajisaka via cnauroth)
+
+ HDFS-5400. DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT constant is set
+ to the wrong value. (Colin Patrick McCabe)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -365,6 +395,9 @@ Release 2.2.1 - UNRELEASED
HDFS-5360. Improvement of usage message of renameSnapshot and
deleteSnapshot. (Shinichi Yamashita via wang)
+ HDFS-5331. make SnapshotDiff.java to a o.a.h.util.Tool interface implementation.
+ (Vinayakumar B via umamahesh)
+
OPTIMIZATIONS
BUG FIXES
@@ -400,6 +433,13 @@ Release 2.2.1 - UNRELEASED
HDFS-5370. Typo in Error Message: different between range in condition
and range in error message. (Kousuke Saruta via suresh)
+ HDFS-5365. Fix libhdfs compile error on FreeBSD9. (Radim Kolar via cnauroth)
+
+ HDFS-5347. Add HDFS NFS user guide. (brandonli)
+
+ HDFS-5403. WebHdfs client cannot communicate with older WebHdfs servers
+ post HDFS-5306. (atm)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Thu Oct 24 21:38:25 2013
@@ -83,7 +83,7 @@
<Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
<Method name="run" />
<Bug pattern="DM_EXIT" />
- </Match>
+ </Match>
<!--
We need to cast objects between old and new api objects
-->
@@ -325,6 +325,12 @@
<Field name="modification" />
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
+ <!-- Replace System.exit() call with ExitUtil.terminate() -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.tools.JMXGet"/>
+ <Method name="main" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH" />
+ </Match>
<Match>
<Class name="org.apache.hadoop.hdfs.server.datanode.ReplicaInfo" />
<Method name="setDirInternal" />
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml Thu Oct 24 21:38:25 2013
@@ -176,6 +176,11 @@ http://maven.apache.org/xsd/maven-4.0.0.
<artifactId>xmlenc</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -542,6 +547,11 @@ http://maven.apache.org/xsd/maven-4.0.0.
<exclude>src/main/docs/releasenotes.html</exclude>
<exclude>src/contrib/**</exclude>
<exclude>src/site/resources/images/*</exclude>
+ <exclude>src/main/webapps/static/dust-full-2.0.0.min.js</exclude>
+ <exclude>src/main/webapps/static/dust-helpers-1.1.1.min.js</exclude>
+ <exclude>src/main/webapps/hdfs/dfshealth.dust.html</exclude>
+ <exclude>src/main/webapps/hdfs/explorer-block-info.dust.html</exclude>
+ <exclude>src/main/webapps/hdfs/explorer.dust.html</exclude>
</excludes>
</configuration>
</plugin>
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Thu Oct 24 21:38:25 2013
@@ -62,6 +62,11 @@ endfunction()
INCLUDE(CheckCSourceCompiles)
CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# Check if we need to link dl library to get dlopen.
+# dlopen on Linux is in separate library but on FreeBSD its in libc
+INCLUDE(CheckLibraryExists)
+CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+
find_package(JNI REQUIRED)
if (NOT GENERATED_JAVAH)
# Must identify where the generated headers have been placed
@@ -89,9 +94,13 @@ add_dual_library(hdfs
main/native/libhdfs/jni_helper.c
main/native/libhdfs/hdfs.c
)
+if (NEED_LINK_DL)
+ set(LIB_DL dl)
+endif(NEED_LINK_DL)
+
target_link_dual_libraries(hdfs
${JAVA_JVM_LIBRARY}
- dl
+ ${LIB_DL}
pthread
)
dual_output_directory(hdfs target/usr/local/lib)
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd Thu Oct 24 21:38:25 2013
@@ -123,6 +123,14 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.GetGroups
goto :eof
+:snapshotDiff
+ set CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
+ goto :eof
+
+:lsSnapshottableDir
+ set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
@@ -164,7 +172,10 @@ goto :eof
@echo fetchdt fetch a delegation token from the NameNode
@echo getconf get config values from configuration
@echo groups get the groups which users belong to
- @echo Use -help to see options
+ @echo snapshotDiff diff two snapshots of a directory or diff the
+ @echo current directory contents with a snapshot
+ @echo lsSnapshottableDir list all snapshottable dirs owned by the current user
+ @echo Use -help to see options
@echo.
@echo Most commands print help when invoked w/o parameters.
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1532946-1535559
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Oct 24 21:38:25 2013
@@ -401,7 +401,7 @@ public class DFSConfigKeys extends Commo
public static final int DFS_CLIENT_MMAP_CACHE_SIZE_DEFAULT = 1024;
public static final String DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS = "dfs.client.mmap.cache.timeout.ms";
public static final long DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS_DEFAULT = 15 * 60 * 1000;
- public static final String DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT = "dfs.client.mmap.cache.timeout.ms";
+ public static final String DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT = "dfs.client.mmap.cache.thread.runs.per.timeout";
public static final int DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT_DEFAULT = 4;
// property for fsimage compression
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Thu Oct 24 21:38:25 2013
@@ -47,7 +47,6 @@ import org.apache.hadoop.fs.FSOutputSumm
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
@@ -138,7 +137,7 @@ public class DFSOutputStream extends FSO
private long currentSeqno = 0;
private long lastQueuedSeqno = -1;
private long lastAckedSeqno = -1;
- private long bytesCurBlock = 0; // bytes writen in current block
+ private long bytesCurBlock = 0; // bytes written in current block
private int packetSize = 0; // write packet size, not including the header.
private int chunksPerPacket = 0;
private final AtomicReference<IOException> lastException = new AtomicReference<IOException>();
@@ -458,8 +457,7 @@ public class DFSOutputStream extends FSO
}
}
- Packet one = null;
-
+ Packet one;
try {
// process datanode IO errors if any
boolean doSleep = false;
@@ -504,7 +502,7 @@ public class DFSOutputStream extends FSO
if(DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Allocating new block");
}
- nodes = nextBlockOutputStream(src);
+ nodes = nextBlockOutputStream();
initDataStreaming();
} else if (stage == BlockConstructionStage.PIPELINE_SETUP_APPEND) {
if(DFSClient.LOG.isDebugEnabled()) {
@@ -569,9 +567,6 @@ public class DFSOutputStream extends FSO
}
lastPacket = Time.now();
- if (one.isHeartbeatPacket()) { //heartbeat packet
- }
-
// update bytesSent
long tmpBytesSent = one.getLastByteOffsetBlock();
if (bytesSent < tmpBytesSent) {
@@ -690,7 +685,7 @@ public class DFSOutputStream extends FSO
}
//
- // Processes reponses from the datanodes. A packet is removed
+ // Processes responses from the datanodes. A packet is removed
// from the ackQueue when its response arrives.
//
private class ResponseProcessor extends Daemon {
@@ -732,18 +727,18 @@ public class DFSOutputStream extends FSO
}
assert seqno != PipelineAck.UNKOWN_SEQNO :
- "Ack for unkown seqno should be a failed ack: " + ack;
+ "Ack for unknown seqno should be a failed ack: " + ack;
if (seqno == Packet.HEART_BEAT_SEQNO) { // a heartbeat ack
continue;
}
// a success ack for a data packet
- Packet one = null;
+ Packet one;
synchronized (dataQueue) {
one = ackQueue.getFirst();
}
if (one.seqno != seqno) {
- throw new IOException("Responseprocessor: Expecting seqno " +
+ throw new IOException("ResponseProcessor: Expecting seqno " +
" for block " + block +
one.seqno + " but received " + seqno);
}
@@ -1052,7 +1047,7 @@ public class DFSOutputStream extends FSO
* Must get block ID and the IDs of the destinations from the namenode.
* Returns the list of target datanodes.
*/
- private DatanodeInfo[] nextBlockOutputStream(String client) throws IOException {
+ private DatanodeInfo[] nextBlockOutputStream() throws IOException {
LocatedBlock lb = null;
DatanodeInfo[] nodes = null;
int count = dfsClient.getConf().nBlockWriteRetry;
@@ -1210,8 +1205,7 @@ public class DFSOutputStream extends FSO
}
private LocatedBlock locateFollowingBlock(long start,
- DatanodeInfo[] excludedNodes)
- throws IOException, UnresolvedLinkException {
+ DatanodeInfo[] excludedNodes) throws IOException {
int retries = dfsClient.getConf().nBlockWriteLocateFollowingRetry;
long sleeptime = 400;
while (true) {
@@ -1283,7 +1277,7 @@ public class DFSOutputStream extends FSO
* Create a socket for a write pipeline
* @param first the first datanode
* @param length the pipeline length
- * @param client
+ * @param client client
* @return the socket connected to the first datanode
*/
static Socket createSocketForPipeline(final DatanodeInfo first,
@@ -1475,7 +1469,7 @@ public class DFSOutputStream extends FSO
//
// Rather than wait around for space in the queue, we should instead try to
// return to the caller as soon as possible, even though we slightly overrun
- // the MAX_PACKETS iength.
+ // the MAX_PACKETS length.
Thread.currentThread().interrupt();
break;
}
@@ -1696,7 +1690,7 @@ public class DFSOutputStream extends FSO
}
}
// If 1) any new blocks were allocated since the last flush, or 2) to
- // update length in NN is requried, then persist block locations on
+ // update length in NN is required, then persist block locations on
// namenode.
if (persistBlocks.getAndSet(false) || updateLength) {
try {
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java Thu Oct 24 21:38:25 2013
@@ -95,18 +95,17 @@ public abstract class BlockPlacementPoli
}
/**
- * Verify that the block is replicated on at least minRacks different racks
- * if there is more than minRacks rack in the system.
+ * Verify if the block's placement meets requirement of placement policy,
+ * i.e. replicas are placed on no less than minRacks racks in the system.
*
* @param srcPath the full pathname of the file to be verified
* @param lBlk block with locations
- * @param minRacks number of racks the block should be replicated to
- * @return the difference between the required and the actual number of racks
- * the block is replicated to.
- */
- abstract public int verifyBlockPlacement(String srcPath,
- LocatedBlock lBlk,
- int minRacks);
+ * @param numOfReplicas replica number of file to be verified
+ * @return the result of verification
+ */
+ abstract public BlockPlacementStatus verifyBlockPlacement(String srcPath,
+ LocatedBlock lBlk,
+ int numOfReplicas);
/**
* Decide whether deleting the specified replica of the block still makes
* the block conform to the configured block placement policy.
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Thu Oct 24 21:38:25 2013
@@ -655,22 +655,22 @@ public class BlockPlacementPolicyDefault
}
@Override
- public int verifyBlockPlacement(String srcPath,
- LocatedBlock lBlk,
- int minRacks) {
+ public BlockPlacementStatus verifyBlockPlacement(String srcPath,
+ LocatedBlock lBlk, int numberOfReplicas) {
DatanodeInfo[] locs = lBlk.getLocations();
if (locs == null)
locs = DatanodeDescriptor.EMPTY_ARRAY;
int numRacks = clusterMap.getNumOfRacks();
if(numRacks <= 1) // only one rack
- return 0;
- minRacks = Math.min(minRacks, numRacks);
+ return new BlockPlacementStatusDefault(
+ Math.min(numRacks, numberOfReplicas), numRacks);
+ int minRacks = Math.min(2, numberOfReplicas);
// 1. Check that all locations are different.
// 2. Count locations on different racks.
Set<String> racks = new TreeSet<String>();
for (DatanodeInfo dn : locs)
racks.add(dn.getNetworkLocation());
- return minRacks - racks.size();
+ return new BlockPlacementStatusDefault(racks.size(), minRacks);
}
@Override
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java Thu Oct 24 21:38:25 2013
@@ -191,6 +191,11 @@ public class DirectoryScanner implements
private final FsVolumeSpi volume;
+ /**
+ * Get the file's length in async block scan
+ */
+ private final long blockFileLength;
+
private final static Pattern CONDENSED_PATH_REGEX =
Pattern.compile("(?<!^)(\\\\|/){2,}");
@@ -235,6 +240,7 @@ public class DirectoryScanner implements
getCondensedPath(vol.getBasePath());
this.blockSuffix = blockFile == null ? null :
getSuffix(blockFile, condensedVolPath);
+ this.blockFileLength = (blockFile != null) ? blockFile.length() : 0;
if (metaFile == null) {
this.metaSuffix = null;
} else if (blockFile == null) {
@@ -251,6 +257,10 @@ public class DirectoryScanner implements
new File(volume.getBasePath(), blockSuffix);
}
+ long getBlockFileLength() {
+ return blockFileLength;
+ }
+
File getMetaFile() {
if (metaSuffix == null) {
return null;
@@ -458,7 +468,7 @@ public class DirectoryScanner implements
// Block metadata file exits and block file is missing
addDifference(diffRecord, statsRecord, info);
} else if (info.getGenStamp() != memBlock.getGenerationStamp()
- || info.getBlockFile().length() != memBlock.getNumBytes()) {
+ || info.getBlockFileLength() != memBlock.getNumBytes()) {
// Block metadata file is missing or has wrong generation stamp,
// or block file length is different than expected
statsRecord.mismatchBlocks++;
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Thu Oct 24 21:38:25 2013
@@ -413,8 +413,15 @@ public class DatanodeWebHdfsMethods {
final long n = length.getValue() != null ?
Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
in.getVisibleLength() - offset.getValue();
- return Response.ok(new OpenEntity(in, n, dfsclient)).type(
- MediaType.APPLICATION_OCTET_STREAM).build();
+
+ /**
+ * Allow the Web UI to perform an AJAX request to get the data.
+ */
+ return Response.ok(new OpenEntity(in, n, dfsclient))
+ .type(MediaType.APPLICATION_OCTET_STREAM)
+ .header("Access-Control-Allow-Methods", "GET")
+ .header("Access-Control-Allow-Origin", "*")
+ .build();
}
case GETFILECHECKSUM:
{
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Oct 24 21:38:25 2013
@@ -270,10 +270,6 @@ public class NameNode implements NameNod
static NameNodeMetrics metrics;
private static final StartupProgress startupProgress = new StartupProgress();
- static {
- StartupProgressMetrics.register(startupProgress);
- }
-
/** Return the {@link FSNamesystem} object.
* @return {@link FSNamesystem} object.
*/
@@ -485,6 +481,7 @@ public class NameNode implements NameNod
loginAsNameNodeUser(conf);
NameNode.initMetrics(conf, this.getRole());
+ StartupProgressMetrics.register(startupProgress);
if (NamenodeRole.NAMENODE == role) {
startHttpServer(conf);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Thu Oct 24 21:38:25 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.net.NetUtils;
@@ -374,9 +375,10 @@ public class NamenodeFsck {
locs.length + " replica(s).");
}
// verify block placement policy
- int missingRacks = BlockPlacementPolicy.getInstance(conf, null, networktopology).
- verifyBlockPlacement(path, lBlk, Math.min(2,targetFileReplication));
- if (missingRacks > 0) {
+ BlockPlacementStatus blockPlacementStatus =
+ BlockPlacementPolicy.getInstance(conf, null, networktopology).
+ verifyBlockPlacement(path, lBlk, targetFileReplication);
+ if (!blockPlacementStatus.isPlacementPolicySatisfied()) {
res.numMisReplicatedBlocks++;
misReplicatedPerFile++;
if (!showFiles) {
@@ -385,9 +387,7 @@ public class NamenodeFsck {
out.print(path + ": ");
}
out.println(" Replica placement policy is violated for " +
- block +
- ". Block should be additionally replicated on " +
- missingRacks + " more rack(s).");
+ block + ". " + blockPlacementStatus.getErrorDescription());
}
report.append(i + ". " + blkName + " len=" + block.getNumBytes());
if (locs.length == 0) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Thu Oct 24 21:38:25 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
import com.google.common.base.Charsets;
@@ -86,7 +87,7 @@ public class DelegationTokenFetcher {
err.println(" --print Print the delegation token");
err.println();
GenericOptionsParser.printGenericCommandUsage(err);
- System.exit(1);
+ ExitUtil.terminate(1);
}
private static Collection<Token<?>> readTokens(Path file, Configuration conf)
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java Thu Oct 24 21:38:25 2013
@@ -43,6 +43,7 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.ExitUtil;
/**
* tool to get data from NameNode or DataNode using MBeans currently the
@@ -295,7 +296,7 @@ public class JMXGet {
// invalid arguments
err("Invalid args");
printUsage(opts);
- System.exit(-1);
+ ExitUtil.terminate(-1);
}
JMXGet jm = new JMXGet();
@@ -317,7 +318,7 @@ public class JMXGet {
if (commandLine.hasOption("help")) {
printUsage(opts);
- System.exit(0);
+ ExitUtil.terminate(0);
}
// rest of args
@@ -342,6 +343,6 @@ public class JMXGet {
res = -1;
}
- System.exit(res);
+ ExitUtil.terminate(res);
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java Thu Oct 24 21:38:25 2013
@@ -20,12 +20,14 @@ package org.apache.hadoop.hdfs.tools.sna
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* A tool used to get the difference report between two snapshots, or between
@@ -38,7 +40,7 @@ import org.apache.hadoop.hdfs.protocol.S
* </pre>
*/
@InterfaceAudience.Private
-public class SnapshotDiff {
+public class SnapshotDiff extends Configured implements Tool {
private static String getSnapshotName(String name) {
if (Path.CUR_DIR.equals(name)) { // current directory
return "";
@@ -57,7 +59,8 @@ public class SnapshotDiff {
return name.substring(i + HdfsConstants.DOT_SNAPSHOT_DIR.length() + 1);
}
- public static void main(String[] argv) throws IOException {
+ @Override
+ public int run(String[] argv) throws Exception {
String description = "SnapshotDiff <snapshotDir> <from> <to>:\n" +
"\tGet the difference between two snapshots, \n" +
"\tor between a snapshot and the current tree of a directory.\n" +
@@ -67,15 +70,14 @@ public class SnapshotDiff {
if(argv.length != 3) {
System.err.println("Usage: \n" + description);
- System.exit(1);
+ return 1;
}
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(getConf());
if (! (fs instanceof DistributedFileSystem)) {
System.err.println(
"SnapshotDiff can only be used in DistributedFileSystem");
- System.exit(1);
+ return 1;
}
DistributedFileSystem dfs = (DistributedFileSystem) fs;
@@ -89,7 +91,14 @@ public class SnapshotDiff {
} catch (IOException e) {
String[] content = e.getLocalizedMessage().split("\n");
System.err.println("snapshotDiff: " + content[0]);
+ return 1;
}
+ return 0;
+ }
+
+ public static void main(String[] argv) throws Exception {
+ int rc = ToolRunner.run(new SnapshotDiff(), argv);
+ System.exit(rc);
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Thu Oct 24 21:38:25 2013
@@ -299,10 +299,15 @@ public class JsonUtil {
}
/** Convert a Json map to an DatanodeInfo object. */
- private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
+ static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
if (m == null) {
return null;
}
+
+ Object infoSecurePort = m.get("infoSecurePort");
+ if (infoSecurePort == null) {
+ infoSecurePort = 0l; // same as the default value in hdfs.proto
+ }
return new DatanodeInfo(
(String)m.get("ipAddr"),
@@ -310,7 +315,7 @@ public class JsonUtil {
(String)m.get("storageID"),
(int)(long)(Long)m.get("xferPort"),
(int)(long)(Long)m.get("infoPort"),
- (int)(long)(Long)m.get("infoSecurePort"),
+ (int)(long)(Long)infoSecurePort,
(int)(long)(Long)m.get("ipcPort"),
(Long)m.get("capacity"),
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java Thu Oct 24 21:38:25 2013
@@ -19,49 +19,114 @@
package org.apache.hadoop.hdfs.web;
import java.io.IOException;
+import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
/**
* Utilities for handling URLs
*/
-@InterfaceAudience.LimitedPrivate({"HDFS"})
+@InterfaceAudience.LimitedPrivate({ "HDFS" })
@InterfaceStability.Unstable
public class URLConnectionFactory {
+ private static final Log LOG = LogFactory.getLog(URLConnectionFactory.class);
+
+ /** SPNEGO authenticator */
+ private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
+
/**
* Timeout for socket connects and reads
*/
- public final static int DEFAULT_SOCKET_TIMEOUT = 1*60*1000; // 1 minute
+ public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
+
+ public static final URLConnectionFactory DEFAULT_CONNECTION_FACTORY = new URLConnectionFactory(
+ DEFAULT_SOCKET_TIMEOUT);
- public static final URLConnectionFactory DEFAULT_CONNECTION_FACTORY = new URLConnectionFactory(DEFAULT_SOCKET_TIMEOUT);
-
private int socketTimeout;
+ /** Configure connections for AuthenticatedURL */
+ private ConnectionConfigurator connConfigurator = new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ URLConnectionFactory.setTimeouts(conn, socketTimeout);
+ return conn;
+ }
+ };
+
public URLConnectionFactory(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
-
+
/**
* Opens a url with read and connect timeouts
- * @param url to open
+ *
+ * @param url
+ * to open
* @return URLConnection
* @throws IOException
*/
public URLConnection openConnection(URL url) throws IOException {
URLConnection connection = url.openConnection();
- setTimeouts(connection);
- return connection;
+ if (connection instanceof HttpURLConnection) {
+ connConfigurator.configure((HttpURLConnection) connection);
+ }
+ return connection;
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url URL to open
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException
+ */
+ public URLConnection openConnection(HttpOpParam.Op op, URL url)
+ throws IOException, AuthenticationException {
+ if (op.getRequireAuth()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("open AuthenticatedURL connection" + url);
+ }
+ UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+ final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+ return new AuthenticatedURL(AUTH, connConfigurator).openConnection(url,
+ authToken);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("open URL connection");
+ }
+ return openConnection(url);
+ }
+ }
+
+ public ConnectionConfigurator getConnConfigurator() {
+ return connConfigurator;
+ }
+
+ public void setConnConfigurator(ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
}
/**
* Sets timeout parameters on the given URLConnection.
*
- * @param connection URLConnection to set
+ * @param connection
+ * URLConnection to set
+ * @param socketTimeout
+ * the connection and read timeout of the connection.
*/
- public void setTimeouts(URLConnection connection) {
+ static void setTimeouts(URLConnection connection, int socketTimeout) {
connection.setConnectTimeout(socketTimeout);
connection.setReadTimeout(socketTimeout);
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Thu Oct 24 21:38:25 2013
@@ -94,9 +94,7 @@ import org.apache.hadoop.ipc.RemoteExcep
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenRenewer;
@@ -119,20 +117,9 @@ public class WebHdfsFileSystem extends F
/** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
- /** SPNEGO authenticator */
- private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
/** Default connection factory may be overridden in tests to use smaller timeout values */
URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY;
- /** Configures connections for AuthenticatedURL */
- private final ConnectionConfigurator CONN_CONFIGURATOR =
- new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- connectionFactory.setTimeouts(conn);
- return conn;
- }
- };
+
/** Delegation token kind */
public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
/** Token selector */
@@ -504,16 +491,7 @@ public class WebHdfsFileSystem extends F
throws IOException {
final HttpURLConnection conn;
try {
- if (op.getRequireAuth()) {
- LOG.debug("open AuthenticatedURL connection");
- UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
- final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
- conn = new AuthenticatedURL(AUTH, CONN_CONFIGURATOR).openConnection(
- url, authToken);
- } else {
- LOG.debug("open URL connection");
- conn = (HttpURLConnection)connectionFactory.openConnection(url);
- }
+ conn = (HttpURLConnection) connectionFactory.openConnection(op, url);
} catch (AuthenticationException e) {
throw new IOException(e);
}
@@ -635,8 +613,10 @@ public class WebHdfsFileSystem extends F
checkRetry = false;
//Step 2) Submit another Http request with the URL from the Location header with data.
- conn = (HttpURLConnection)connectionFactory.openConnection(new URL(redirect));
- conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
+ conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
+ redirect));
+ conn.setRequestProperty("Content-Type",
+ MediaType.APPLICATION_OCTET_STREAM);
conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
connect();
return conn;
@@ -658,7 +638,8 @@ public class WebHdfsFileSystem extends F
disconnect();
checkRetry = false;
- conn = (HttpURLConnection)connectionFactory.openConnection(new URL(redirect));
+ conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
+ redirect));
connect();
}
@@ -892,12 +873,6 @@ public class WebHdfsFileSystem extends F
.write(bufferSize);
}
- @SuppressWarnings("deprecation")
- @Override
- public boolean delete(final Path f) throws IOException {
- return delete(f, true);
- }
-
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1532946-1535559
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/posix_util.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/posix_util.c?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/posix_util.c (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/posix_util.c Thu Oct 24 21:38:25 2013
@@ -27,6 +27,7 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
+#include <limits.h>
static pthread_mutex_t gTempdirLock = PTHREAD_MUTEX_INITIALIZER;
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1532946-1535559
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1532946-1535559
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1532946-1535559
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1532946-1535559
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Thu Oct 24 21:38:25 2013
@@ -83,7 +83,6 @@ import org.apache.log4j.RollingFileAppen
import org.junit.Test;
import com.google.common.collect.Sets;
-import org.mockito.Mockito;
import static org.mockito.Mockito.*;
/**
@@ -892,6 +891,80 @@ public class TestFsck {
}
}
}
+
+ /**
+ * Tests that the # of misreplaced replicas is correct
+ * @throws IOException
+ */
+ @Test
+ public void testFsckMisPlacedReplicas() throws IOException {
+ // Desired replication factor
+ final short REPL_FACTOR = 2;
+ // Number of replicas to actually start
+ short NUM_DN = 2;
+ // Number of blocks to write
+ final short NUM_BLOCKS = 3;
+ // Set a small-ish blocksize
+ final long blockSize = 512;
+
+ String [] racks = {"/rack1", "/rack1"};
+ String [] hosts = {"host1", "host2"};
+
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+
+ MiniDFSCluster cluster = null;
+ DistributedFileSystem dfs = null;
+
+ try {
+ // Startup a minicluster
+ cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts)
+ .racks(racks).build();
+ assertNotNull("Failed Cluster Creation", cluster);
+ cluster.waitClusterUp();
+ dfs = (DistributedFileSystem) cluster.getFileSystem();
+ assertNotNull("Failed to get FileSystem", dfs);
+
+ // Create a file that will be intentionally under-replicated
+ final String pathString = new String("/testfile");
+ final Path path = new Path(pathString);
+ long fileLen = blockSize * NUM_BLOCKS;
+ DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1);
+
+ // Create an under-replicated file
+ NameNode namenode = cluster.getNameNode();
+ NetworkTopology nettop = cluster.getNamesystem().getBlockManager()
+ .getDatanodeManager().getNetworkTopology();
+ // Add a new node on different rack, so previous blocks' replicas
+ // are considered to be misplaced
+ nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3"));
+ NUM_DN++;
+
+ Map<String,String[]> pmap = new HashMap<String, String[]>();
+ Writer result = new StringWriter();
+ PrintWriter out = new PrintWriter(result, true);
+ InetAddress remoteAddress = InetAddress.getLocalHost();
+ NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
+ NUM_DN, (short)REPL_FACTOR, remoteAddress);
+
+ // Run the fsck and check the Result
+ final HdfsFileStatus file =
+ namenode.getRpcServer().getFileInfo(pathString);
+ assertNotNull(file);
+ Result res = new Result(conf);
+ fsck.check(pathString, file, res);
+ // check misReplicatedBlock number.
+ assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS);
+ } finally {
+ if(dfs != null) {
+ dfs.close();
+ }
+ if(cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/** Test fsck with FileNotFound */
@Test
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Thu Oct 24 21:38:25 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.web;
+import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.FileStatus;
@@ -58,4 +59,27 @@ public class TestJsonUtil {
System.out.println("fs2 = " + fs2);
Assert.assertEquals(fstatus, fs2);
}
+
+ @Test
+ public void testToDatanodeInfoWithoutSecurePort() {
+ Map<String, Object> response = new HashMap<String, Object>();
+
+ response.put("ipAddr", "127.0.0.1");
+ response.put("hostName", "localhost");
+ response.put("storageID", "fake-id");
+ response.put("xferPort", 1337l);
+ response.put("infoPort", 1338l);
+ // deliberately don't include an entry for "infoSecurePort"
+ response.put("ipcPort", 1339l);
+ response.put("capacity", 1024l);
+ response.put("dfsUsed", 512l);
+ response.put("remaining", 512l);
+ response.put("blockPoolUsed", 512l);
+ response.put("lastUpdate", 0l);
+ response.put("xceiverCount", 4096l);
+ response.put("networkLocation", "foo.bar.baz");
+ response.put("adminState", "NORMAL");
+
+ JsonUtil.toDatanodeInfo(response);
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java Thu Oct 24 21:38:25 2013
@@ -25,9 +25,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.net.SocketAddress;
import java.net.SocketTimeoutException;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
@@ -41,6 +43,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -71,8 +74,9 @@ public class TestWebHdfsTimeouts {
@Before
public void setUp() throws Exception {
Configuration conf = WebHdfsTestUtil.createConf();
- nnHttpAddress = NameNode.getHttpAddress(conf);
- serverSocket = new ServerSocket(nnHttpAddress.getPort(), CONNECTION_BACKLOG);
+ serverSocket = new ServerSocket(0, CONNECTION_BACKLOG);
+ nnHttpAddress = new InetSocketAddress("localhost", serverSocket.getLocalPort());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "localhost:" + serverSocket.getLocalPort());
fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
fs.connectionFactory = connectionFactory;
clients = new ArrayList<SocketChannel>();
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java Thu Oct 24 21:38:25 2013
@@ -21,9 +21,13 @@ package org.apache.hadoop.tools;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.io.PrintStream;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
@@ -92,6 +96,7 @@ public class TestJMXGet {
//jmx.init();
//jmx = new JMXGet();
jmx.init(); // default lists namenode mbeans only
+ assertTrue("error printAllValues", checkPrintAllValues(jmx));
//get some data from different source
assertEquals(numDatanodes, Integer.parseInt(
@@ -103,7 +108,24 @@ public class TestJMXGet {
cluster.shutdown();
}
-
+
+ private static boolean checkPrintAllValues(JMXGet jmx) throws Exception {
+ int size = 0;
+ byte[] bytes = null;
+ String pattern = "List of all the available keys:";
+ PipedOutputStream pipeOut = new PipedOutputStream();
+ PipedInputStream pipeIn = new PipedInputStream(pipeOut);
+ System.setErr(new PrintStream(pipeOut));
+ jmx.printAllValues();
+ if ((size = pipeIn.available()) != 0) {
+ bytes = new byte[size];
+ pipeIn.read(bytes, 0, bytes.length);
+ }
+ pipeOut.close();
+ pipeIn.close();
+ return bytes != null ? new String(bytes).contains(pattern) : false;
+ }
+
/**
* test JMX connection to DataNode..
* @throws Exception