You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/08/15 21:04:28 UTC
svn commit: r1373573 [1/2] - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/
src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache...
Author: todd
Date: Wed Aug 15 19:04:24 2012
New Revision: 1373573
URL: http://svn.apache.org/viewvc?rev=1373573&view=rev
Log:
Merge trunk into branch
Modified:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1372630-1373572
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Aug 15 19:04:24 2012
@@ -114,6 +114,12 @@ Trunk (unreleased changes)
HDFS-3789. JournalManager#format() should be able to throw IOException
(Ivan Kelly via todd)
+ HDFS-3723. Add support -h, -help to all the commands. (Jing Zhao via
+ suresh)
+
+ HDFS-3803. Change BlockPoolSliceScanner chatty INFO log to DEBUG.
+ (Andrew Purtell via suresh)
+
OPTIMIZATIONS
BUG FIXES
@@ -183,6 +189,8 @@ Trunk (unreleased changes)
HDFS-3625. Fix TestBackupNode by properly initializing edit log during
startup. (Junping Du via todd)
+ HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd)
+
Branch-2 ( Unreleased changes )
INCOMPATIBLE CHANGES
@@ -209,6 +217,8 @@ Branch-2 ( Unreleased changes )
HDFS-3637. Add support for encrypting the DataTransferProtocol. (atm)
+ HDFS-3150. Add option for clients to contact DNs via hostname. (eli)
+
IMPROVEMENTS
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
@@ -381,6 +391,14 @@ Branch-2 ( Unreleased changes )
HDFS-3276. initializeSharedEdits should have a -nonInteractive flag (todd)
+ HDFS-3765. namenode -initializeSharedEdits should be able to initialize
+ all shared storages. (Vinay and todd via todd)
+
+ HDFS-3802. StartupOption.name in HdfsServerConstants should be final.
+ (Jing Zhao via szetszwo)
+
+ HDFS-3796. Speed up edit log tests by avoiding fsync() (todd)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
@@ -587,6 +605,15 @@ Branch-2 ( Unreleased changes )
IOExceptions of stream closures can mask root exceptions. (Uma Maheswara
Rao G via szetszwo)
+ HDFS-3790. test_fuse_dfs.c doesn't compile on centos 5. (Colin Patrick
+ McCabe via atm)
+
+ HDFS-3658. Fix bugs in TestDFSClientRetries and add more tests. (szetszwo)
+
+ HDFS-3794. WebHDFS OPEN returns the incorrect Content-Length in the HTTP
+ header when offset is specified and length is omitted.
+ (Ravi Prakash via szetszwo)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
@@ -1462,6 +1489,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3553. Hftp proxy tokens are broken (daryn)
+ HDFS-3718. Datanode won't shutdown because of runaway DataBlockScanner
+ thread (Kihwal Lee via daryn)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java Wed Aug 15 19:04:24 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.contrib.bkjournal;
import static org.junit.Assert.*;
+
import org.junit.Test;
import org.junit.Before;
import org.junit.After;
@@ -25,6 +26,9 @@ import org.junit.BeforeClass;
import org.junit.AfterClass;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.ServiceFailedException;
+import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
+import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HAUtil;
@@ -35,12 +39,16 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.bookkeeper.proto.BookieServer;
@@ -48,7 +56,9 @@ import org.apache.bookkeeper.proto.Booki
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import java.io.File;
import java.io.IOException;
+import java.net.URISyntaxException;
/**
* Integration test to ensure that the BookKeeper JournalManager
@@ -67,6 +77,11 @@ public class TestBookKeeperAsHASharedDir
bkutil = new BKJMUtil(numBookies);
bkutil.start();
}
+
+ @Before
+ public void clearExitStatus() {
+ ExitUtil.resetFirstExitException();
+ }
@AfterClass
public static void teardownBookkeeper() throws Exception {
@@ -244,4 +259,97 @@ public class TestBookKeeperAsHASharedDir
}
}
}
+
+ /**
+ * Use NameNode INTIALIZESHAREDEDITS to initialize the shared edits. i.e. copy
+ * the edits log segments to new bkjm shared edits.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testInitializeBKSharedEdits() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ Configuration conf = new Configuration();
+ HAUtil.setAllowStandbyReads(conf, true);
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+
+ MiniDFSNNTopology topology = MiniDFSNNTopology.simpleHATopology();
+ cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology)
+ .numDataNodes(0).build();
+ cluster.waitActive();
+ // Shutdown and clear the current filebased shared dir.
+ cluster.shutdownNameNodes();
+ File shareddir = new File(cluster.getSharedEditsDir(0, 1));
+ assertTrue("Initial Shared edits dir not fully deleted",
+ FileUtil.fullyDelete(shareddir));
+
+ // Check namenodes should not start without shared dir.
+ assertCanNotStartNamenode(cluster, 0);
+ assertCanNotStartNamenode(cluster, 1);
+
+ // Configure bkjm as new shared edits dir in both namenodes
+ Configuration nn1Conf = cluster.getConfiguration(0);
+ Configuration nn2Conf = cluster.getConfiguration(1);
+ nn1Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil
+ .createJournalURI("/initializeSharedEdits").toString());
+ nn2Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil
+ .createJournalURI("/initializeSharedEdits").toString());
+ BKJMUtil.addJournalManagerDefinition(nn1Conf);
+ BKJMUtil.addJournalManagerDefinition(nn2Conf);
+
+ // Initialize the BKJM shared edits.
+ assertFalse(NameNode.initializeSharedEdits(nn1Conf));
+
+ // NameNode should be able to start and should be in sync with BKJM as
+ // shared dir
+ assertCanStartHANameNodes(cluster, conf, "/testBKJMInitialize");
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private void assertCanNotStartNamenode(MiniDFSCluster cluster, int nnIndex) {
+ try {
+ cluster.restartNameNode(nnIndex, false);
+ fail("Should not have been able to start NN" + (nnIndex)
+ + " without shared dir");
+ } catch (IOException ioe) {
+ LOG.info("Got expected exception", ioe);
+ GenericTestUtils.assertExceptionContains(
+ "Cannot start an HA namenode with name dirs that need recovery", ioe);
+ }
+ }
+
+ private void assertCanStartHANameNodes(MiniDFSCluster cluster,
+ Configuration conf, String path) throws ServiceFailedException,
+ IOException, URISyntaxException, InterruptedException {
+ // Now should be able to start both NNs. Pass "false" here so that we don't
+ // try to waitActive on all NNs, since the second NN doesn't exist yet.
+ cluster.restartNameNode(0, false);
+ cluster.restartNameNode(1, true);
+
+ // Make sure HA is working.
+ cluster
+ .getNameNode(0)
+ .getRpcServer()
+ .transitionToActive(
+ new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER));
+ FileSystem fs = null;
+ try {
+ Path newPath = new Path(path);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(newPath));
+ HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
+ cluster.getNameNode(1));
+ assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
+ newPath.toString(), false).isDir());
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+ }
}
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1372630-1373572
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java Wed Aug 15 19:04:24 2012
@@ -86,11 +86,11 @@ class BlockReaderLocal implements BlockR
}
private synchronized ClientDatanodeProtocol getDatanodeProxy(
- DatanodeInfo node, Configuration conf, int socketTimeout)
- throws IOException {
+ DatanodeInfo node, Configuration conf, int socketTimeout,
+ boolean connectToDnViaHostname) throws IOException {
if (proxy == null) {
proxy = DFSUtil.createClientDatanodeProtocolProxy(node, conf,
- socketTimeout);
+ socketTimeout, connectToDnViaHostname);
}
return proxy;
}
@@ -156,14 +156,16 @@ class BlockReaderLocal implements BlockR
*/
static BlockReaderLocal newBlockReader(Configuration conf, String file,
ExtendedBlock blk, Token<BlockTokenIdentifier> token, DatanodeInfo node,
- int socketTimeout, long startOffset, long length) throws IOException {
+ int socketTimeout, long startOffset, long length,
+ boolean connectToDnViaHostname) throws IOException {
LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node
.getIpcPort());
// check the cache first
BlockLocalPathInfo pathinfo = localDatanodeInfo.getBlockLocalPathInfo(blk);
if (pathinfo == null) {
- pathinfo = getBlockPathInfo(blk, node, conf, socketTimeout, token);
+ pathinfo = getBlockPathInfo(blk, node, conf, socketTimeout, token,
+ connectToDnViaHostname);
}
// check to see if the file exists. It may so happen that the
@@ -241,11 +243,12 @@ class BlockReaderLocal implements BlockR
private static BlockLocalPathInfo getBlockPathInfo(ExtendedBlock blk,
DatanodeInfo node, Configuration conf, int timeout,
- Token<BlockTokenIdentifier> token) throws IOException {
+ Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname)
+ throws IOException {
LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.getIpcPort());
BlockLocalPathInfo pathinfo = null;
ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(node,
- conf, timeout);
+ conf, timeout, connectToDnViaHostname);
try {
// make RPC to local datanode to find local pathnames of blocks
pathinfo = proxy.getBlockLocalPathInfo(blk, token);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Wed Aug 15 19:04:24 2012
@@ -49,6 +49,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
@@ -213,6 +215,7 @@ public class DFSClient implements java.i
final String taskId;
final FsPermission uMask;
final boolean useLegacyBlockReader;
+ final boolean connectToDnViaHostname;
Conf(Configuration conf) {
maxFailoverAttempts = conf.getInt(
@@ -263,6 +266,8 @@ public class DFSClient implements java.i
useLegacyBlockReader = conf.getBoolean(
DFS_CLIENT_USE_LEGACY_BLOCKREADER,
DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT);
+ connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
+ DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
}
private int getChecksumType(Configuration conf) {
@@ -473,6 +478,14 @@ public class DFSClient implements java.i
return clientName;
}
+ /**
+ * @return whether the client should use hostnames instead of IPs
+ * when connecting to DataNodes
+ */
+ boolean connectToDnViaHostname() {
+ return dfsClientConf.connectToDnViaHostname;
+ }
+
void checkOpen() throws IOException {
if (!clientRunning) {
IOException result = new IOException("Filesystem closed");
@@ -729,12 +742,12 @@ public class DFSClient implements java.i
*/
static BlockReader getLocalBlockReader(Configuration conf,
String src, ExtendedBlock blk, Token<BlockTokenIdentifier> accessToken,
- DatanodeInfo chosenNode, int socketTimeout, long offsetIntoBlock)
- throws InvalidToken, IOException {
+ DatanodeInfo chosenNode, int socketTimeout, long offsetIntoBlock,
+ boolean connectToDnViaHostname) throws InvalidToken, IOException {
try {
return BlockReaderLocal.newBlockReader(conf, src, blk, accessToken,
chosenNode, socketTimeout, offsetIntoBlock, blk.getNumBytes()
- - offsetIntoBlock);
+ - offsetIntoBlock, connectToDnViaHostname);
} catch (RemoteException re) {
throw re.unwrapRemoteException(InvalidToken.class,
AccessControlException.class);
@@ -1425,7 +1438,8 @@ public class DFSClient implements java.i
public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
checkOpen();
return getFileChecksum(src, namenode, socketFactory,
- dfsClientConf.socketTimeout, getDataEncryptionKey());
+ dfsClientConf.socketTimeout, getDataEncryptionKey(),
+ dfsClientConf.connectToDnViaHostname);
}
@InterfaceAudience.Private
@@ -1471,7 +1485,8 @@ public class DFSClient implements java.i
*/
public static MD5MD5CRC32FileChecksum getFileChecksum(String src,
ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
- DataEncryptionKey encryptionKey) throws IOException {
+ DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
+ throws IOException {
//get all block locations
LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
if (null == blockLocations) {
@@ -1509,9 +1524,11 @@ public class DFSClient implements java.i
try {
//connect to a datanode
sock = socketFactory.createSocket();
- NetUtils.connect(sock,
- NetUtils.createSocketAddr(datanodes[j].getXferAddr()),
- timeout);
+ String dnAddr = datanodes[j].getXferAddr(connectToDnViaHostname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr);
+ }
+ NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
sock.setSoTimeout(timeout);
OutputStream unbufOut = NetUtils.getOutputStream(sock);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Wed Aug 15 19:04:24 2012
@@ -52,6 +52,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_CLIENT_WRITE_REPLACE_DATANODE_ON_FAILURE_POLICY_DEFAULT = "DEFAULT";
public static final String DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY = "dfs.client.socketcache.capacity";
public static final int DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT = 16;
+ public static final String DFS_CLIENT_USE_DN_HOSTNAME = "dfs.client.use.datanode.hostname";
+ public static final boolean DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT = false;
// HA related configuration
public static final String DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX = "dfs.client.failover.proxy.provider";
@@ -81,6 +83,8 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_DATANODE_SYNC_BEHIND_WRITES_DEFAULT = false;
public static final String DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY = "dfs.datanode.drop.cache.behind.reads";
public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false;
+ public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname";
+ public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false;
public static final String DFS_NAMENODE_HTTP_PORT_KEY = "dfs.http.port";
public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Wed Aug 15 19:04:24 2012
@@ -199,7 +199,8 @@ public class DFSInputStream extends FSIn
try {
cdp = DFSUtil.createClientDatanodeProtocolProxy(
- datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);
+ datanode, dfsClient.conf, dfsClient.getConf().socketTimeout,
+ dfsClient.getConf().connectToDnViaHostname, locatedblock);
final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());
@@ -716,8 +717,12 @@ public class DFSInputStream extends FSIn
DatanodeInfo[] nodes = block.getLocations();
try {
DatanodeInfo chosenNode = bestNode(nodes, deadNodes);
- InetSocketAddress targetAddr =
- NetUtils.createSocketAddr(chosenNode.getXferAddr());
+ final String dnAddr =
+ chosenNode.getXferAddr(dfsClient.connectToDnViaHostname());
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
+ }
+ InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr);
return new DNAddrPair(chosenNode, targetAddr);
} catch (IOException ie) {
String blockInfo = block.getBlock() + " file=" + src;
@@ -875,7 +880,8 @@ public class DFSInputStream extends FSIn
if (dfsClient.shouldTryShortCircuitRead(dnAddr)) {
return DFSClient.getLocalBlockReader(dfsClient.conf, src, block,
- blockToken, chosenNode, dfsClient.hdfsTimeout, startOffset);
+ blockToken, chosenNode, dfsClient.hdfsTimeout, startOffset,
+ dfsClient.connectToDnViaHostname());
}
IOException err = null;
@@ -1183,7 +1189,7 @@ public class DFSInputStream extends FSIn
throw new IOException("No live nodes contain current block");
}
- /** Utility class to encapsulate data node info and its ip address. */
+ /** Utility class to encapsulate data node info and its address. */
static class DNAddrPair {
DatanodeInfo info;
InetSocketAddress addr;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Wed Aug 15 19:04:24 2012
@@ -1100,7 +1100,7 @@ public class DFSOutputStream extends FSO
if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ "encryption key was invalid when connecting to "
- + nodes[0].getXferAddr() + " : " + ie);
+ + nodes[0] + " : " + ie);
// The encryption key used is invalid.
refetchEncryptionKey--;
dfsClient.clearDataEncryptionKey();
@@ -1112,7 +1112,8 @@ public class DFSOutputStream extends FSO
// find the datanode that matches
if (firstBadLink.length() != 0) {
for (int i = 0; i < nodes.length; i++) {
- if (nodes[i].getXferAddr().equals(firstBadLink)) {
+ // NB: Unconditionally using the xfer addr w/o hostname
+ if (firstBadLink.equals(nodes[i].getXferAddr())) {
errorIndex = i;
break;
}
@@ -1216,11 +1217,11 @@ public class DFSOutputStream extends FSO
*/
static Socket createSocketForPipeline(final DatanodeInfo first,
final int length, final DFSClient client) throws IOException {
- if(DFSClient.LOG.isDebugEnabled()) {
- DFSClient.LOG.debug("Connecting to datanode " + first);
+ final String dnAddr = first.getXferAddr(client.connectToDnViaHostname());
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
}
- final InetSocketAddress isa =
- NetUtils.createSocketAddr(first.getXferAddr());
+ final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
final Socket sock = client.socketFactory.createSocket();
final int timeout = client.getDatanodeReadTimeout(length);
NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), timeout);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Wed Aug 15 19:04:24 2012
@@ -18,8 +18,21 @@
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+
import java.io.IOException;
+import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -33,10 +46,17 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import java.util.StringTokenizer;
import javax.net.SocketFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -57,8 +77,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
@@ -424,7 +443,6 @@ public class DFSUtil {
*
* @param conf configuration
* @return list of InetSocketAddresses
- * @throws IOException if no addresses are configured
*/
public static Map<String, Map<String, InetSocketAddress>> getHaNnRpcAddresses(
Configuration conf) {
@@ -841,17 +859,17 @@ public class DFSUtil {
/** Create a {@link ClientDatanodeProtocol} proxy */
public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
DatanodeID datanodeid, Configuration conf, int socketTimeout,
- LocatedBlock locatedBlock) throws IOException {
+ boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout,
- locatedBlock);
+ connectToDnViaHostname, locatedBlock);
}
/** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */
static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
- DatanodeID datanodeid, Configuration conf, int socketTimeout)
- throws IOException {
+ DatanodeID datanodeid, Configuration conf, int socketTimeout,
+ boolean connectToDnViaHostname) throws IOException {
return new ClientDatanodeProtocolTranslatorPB(
- datanodeid, conf, socketTimeout);
+ datanodeid, conf, socketTimeout, connectToDnViaHostname);
}
/** Create a {@link ClientDatanodeProtocol} proxy */
@@ -1073,4 +1091,44 @@ public class DFSUtil {
return null;
}
}
+
+ public static Options helpOptions = new Options();
+ public static Option helpOpt = new Option("h", "help", false,
+ "get help information");
+
+ static {
+ helpOptions.addOption(helpOpt);
+ }
+
+ /**
+ * Parse the arguments for commands
+ *
+ * @param args the argument to be parsed
+ * @param helpDescription help information to be printed out
+ * @param out Printer
+ * @param printGenericCommandUsage whether to print the
+ * generic command usage defined in ToolRunner
+ * @return true when the argument matches help option, false if not
+ */
+ public static boolean parseHelpArgument(String[] args,
+ String helpDescription, PrintStream out, boolean printGenericCommandUsage) {
+ if (args.length == 1) {
+ try {
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmdLine = parser.parse(helpOptions, args);
+ if (cmdLine.hasOption(helpOpt.getOpt())
+ || cmdLine.hasOption(helpOpt.getLongOpt())) {
+ // should print out the help information
+ out.println(helpDescription + "\n");
+ if (printGenericCommandUsage) {
+ ToolRunner.printGenericCommandUsage(out);
+ }
+ return true;
+ }
+ } catch (ParseException pe) {
+ return false;
+ }
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Wed Aug 15 19:04:24 2012
@@ -104,7 +104,7 @@ public class DatanodeID implements Compa
/**
* @return IP:ipcPort string
*/
- public String getIpcAddr() {
+ private String getIpcAddr() {
return ipAddr + ":" + ipcPort;
}
@@ -123,6 +123,29 @@ public class DatanodeID implements Compa
}
/**
+ * @return hostname:ipcPort
+ */
+ private String getIpcAddrWithHostname() {
+ return hostName + ":" + ipcPort;
+ }
+
+ /**
+ * @param useHostname true to use the DN hostname, use the IP otherwise
+ * @return name:xferPort
+ */
+ public String getXferAddr(boolean useHostname) {
+ return useHostname ? getXferAddrWithHostname() : getXferAddr();
+ }
+
+ /**
+ * @param useHostname true to use the DN hostname, use the IP otherwise
+ * @return name:ipcPort
+ */
+ public String getIpcAddr(boolean useHostname) {
+ return useHostname ? getIpcAddrWithHostname() : getIpcAddr();
+ }
+
+ /**
* @return data storage ID.
*/
public String getStorageID() {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java Wed Aug 15 19:04:24 2012
@@ -73,10 +73,10 @@ public class ClientDatanodeProtocolTrans
RefreshNamenodesRequestProto.newBuilder().build();
public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid,
- Configuration conf, int socketTimeout, LocatedBlock locatedBlock)
- throws IOException {
+ Configuration conf, int socketTimeout, boolean connectToDnViaHostname,
+ LocatedBlock locatedBlock) throws IOException {
rpcProxy = createClientDatanodeProtocolProxy( datanodeid, conf,
- socketTimeout, locatedBlock);
+ socketTimeout, connectToDnViaHostname, locatedBlock);
}
public ClientDatanodeProtocolTranslatorPB(InetSocketAddress addr,
@@ -90,11 +90,17 @@ public class ClientDatanodeProtocolTrans
* @param datanodeid Datanode to connect to.
* @param conf Configuration.
* @param socketTimeout Socket timeout to use.
+ * @param connectToDnViaHostname connect to the Datanode using its hostname
* @throws IOException
*/
public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid,
- Configuration conf, int socketTimeout) throws IOException {
- InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getIpcAddr());
+ Configuration conf, int socketTimeout, boolean connectToDnViaHostname)
+ throws IOException {
+ final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
+ InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
+ }
rpcProxy = createClientDatanodeProtocolProxy(addr,
UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf), socketTimeout);
@@ -102,10 +108,11 @@ public class ClientDatanodeProtocolTrans
static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
DatanodeID datanodeid, Configuration conf, int socketTimeout,
- LocatedBlock locatedBlock) throws IOException {
- InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getIpcAddr());
+ boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
+ final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
+ InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
if (LOG.isDebugEnabled()) {
- LOG.debug("ClientDatanodeProtocol addr=" + addr);
+ LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Wed Aug 15 19:04:24 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.balancer;
+import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.hdfs.protocol.HdfsProtoUtil.vintPrefixed;
import java.io.BufferedInputStream;
@@ -26,6 +27,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.Socket;
import java.net.URI;
import java.text.DateFormat;
@@ -68,7 +70,6 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.io.IOUtils;
@@ -79,7 +80,6 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import static com.google.common.base.Preconditions.checkArgument;
/** <p>The balancer is a tool that balances disk space usage on an HDFS cluster
* when some datanodes become full or when new empty nodes join the cluster.
@@ -189,6 +189,13 @@ public class Balancer {
*/
public static final int MAX_NUM_CONCURRENT_MOVES = 5;
+ private static final String USAGE = "Usage: java "
+ + Balancer.class.getSimpleName()
+ + "\n\t[-policy <policy>]\tthe balancing policy: "
+ + BalancingPolicy.Node.INSTANCE.getName() + " or "
+ + BalancingPolicy.Pool.INSTANCE.getName()
+ + "\n\t[-threshold <threshold>]\tPercentage of disk capacity";
+
private final NameNodeConnector nnc;
private final BalancingPolicy policy;
private final double threshold;
@@ -1550,7 +1557,7 @@ public class Balancer {
}
}
} catch(RuntimeException e) {
- printUsage();
+ printUsage(System.err);
throw e;
}
}
@@ -1558,13 +1565,8 @@ public class Balancer {
return new Parameters(policy, threshold);
}
- private static void printUsage() {
- System.out.println("Usage: java " + Balancer.class.getSimpleName());
- System.out.println(" [-policy <policy>]\tthe balancing policy: "
- + BalancingPolicy.Node.INSTANCE.getName() + " or "
- + BalancingPolicy.Pool.INSTANCE.getName());
- System.out.println(
- " [-threshold <threshold>]\tPercentage of disk capacity");
+ private static void printUsage(PrintStream out) {
+ out.println(USAGE + "\n");
}
}
@@ -1573,6 +1575,10 @@ public class Balancer {
* @param args Command line arguments
*/
public static void main(String[] args) {
+ if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
try {
System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args));
} catch (Throwable e) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java Wed Aug 15 19:04:24 2012
@@ -61,7 +61,7 @@ public final class HdfsServerConstants {
FORCE("-force"),
NONINTERACTIVE("-nonInteractive");
- private String name = null;
+ private final String name;
// Used only with format and upgrade options
private String clusterId = null;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java Wed Aug 15 19:04:24 2012
@@ -554,9 +554,11 @@ class BlockPoolSliceScanner {
}
private synchronized void startNewPeriod() {
- LOG.info("Starting a new period : work left in prev period : "
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting a new period : work left in prev period : "
+ String.format("%.2f%%", totalBytesToScan == 0 ? 0
: (bytesLeft * 100.0) / totalBytesToScan));
+ }
// reset the byte counts :
bytesLeft = totalBytesToScan;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java Wed Aug 15 19:04:24 2012
@@ -55,7 +55,7 @@ class DNConf {
final boolean dropCacheBehindReads;
final boolean syncOnClose;
final boolean encryptDataTransfer;
-
+ final boolean connectToDnViaHostname;
final long readaheadLength;
final long heartBeatInterval;
@@ -97,7 +97,9 @@ class DNConf {
dropCacheBehindReads = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY,
DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT);
-
+ connectToDnViaHostname = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
this.blockReportInterval = conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Wed Aug 15 19:04:24 2012
@@ -46,6 +46,7 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTPS_ENABLE_KEY;
+import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
@@ -55,6 +56,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -98,8 +100,8 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
-import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor;
+import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService;
@@ -124,9 +126,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.Util;
-
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
@@ -171,9 +170,9 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.mortbay.util.ajax.JSON;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
/**********************************************************
@@ -230,6 +229,8 @@ public class DataNode extends Configured
static final Log ClientTraceLog =
LogFactory.getLog(DataNode.class.getName() + ".clienttrace");
+
+ private static final String USAGE = "Usage: java DataNode [-rollback | -regular]";
/**
* Use {@link NetUtils#createSocketAddr(String)} instead.
@@ -276,6 +277,7 @@ public class DataNode extends Configured
private Configuration conf;
private final String userWithLocalPathAccess;
+ private boolean connectToDnViaHostname;
ReadaheadPool readaheadPool;
/**
@@ -296,8 +298,11 @@ public class DataNode extends Configured
final SecureResources resources) throws IOException {
super(conf);
- this.userWithLocalPathAccess = conf
- .get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
+ this.userWithLocalPathAccess =
+ conf.get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
+ this.connectToDnViaHostname = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
try {
hostName = getHostName(conf);
LOG.info("Configured hostname is " + hostName);
@@ -878,7 +883,7 @@ public class DataNode extends Configured
/**
* NB: The datanode can perform data transfer on the streaming
* address however clients are given the IPC IP address for data
- * transfer, and that may be be a different address.
+ * transfer, and that may be a different address.
*
* @return socket address for data transfer
*/
@@ -925,12 +930,12 @@ public class DataNode extends Configured
}
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
- DatanodeID datanodeid, final Configuration conf, final int socketTimeout)
- throws IOException {
- final InetSocketAddress addr =
- NetUtils.createSocketAddr(datanodeid.getIpcAddr());
- if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
- InterDatanodeProtocol.LOG.debug("InterDatanodeProtocol addr=" + addr);
+ DatanodeID datanodeid, final Configuration conf, final int socketTimeout,
+ final boolean connectToDnViaHostname) throws IOException {
+ final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
+ final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();
try {
@@ -1061,6 +1066,7 @@ public class DataNode extends Configured
}
}
+ this.shouldRun = false;
shutdownPeriodicScanners();
if (infoServer != null) {
@@ -1074,7 +1080,6 @@ public class DataNode extends Configured
ipcServer.stop();
}
- this.shouldRun = false;
if (dataXceiverServer != null) {
((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();
this.dataXceiverServer.interrupt();
@@ -1386,8 +1391,11 @@ public class DataNode extends Configured
final boolean isClient = clientname.length() > 0;
try {
- InetSocketAddress curTarget =
- NetUtils.createSocketAddr(targets[0].getXferAddr());
+ final String dnAddr = targets[0].getXferAddr(connectToDnViaHostname);
+ InetSocketAddress curTarget = NetUtils.createSocketAddr(dnAddr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr);
+ }
sock = newSocket();
NetUtils.connect(sock, curTarget, dnConf.socketTimeout);
sock.setSoTimeout(targets.length * dnConf.socketTimeout);
@@ -1534,7 +1542,7 @@ public class DataNode extends Configured
}
if (!parseArguments(args, conf)) {
- printUsage();
+ printUsage(System.err);
return null;
}
Collection<URI> dataDirs = getStorageDirs(conf);
@@ -1648,9 +1656,8 @@ public class DataNode extends Configured
+ xmitsInProgress.get() + "}";
}
- private static void printUsage() {
- System.err.println("Usage: java DataNode");
- System.err.println(" [-rollback]");
+ private static void printUsage(PrintStream out) {
+ out.println(USAGE + "\n");
}
/**
@@ -1735,6 +1742,10 @@ public class DataNode extends Configured
}
public static void main(String args[]) {
+ if (DFSUtil.parseHelpArgument(args, DataNode.USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
secureMain(args, null);
}
@@ -1843,7 +1854,7 @@ public class DataNode extends Configured
DatanodeRegistration bpReg = bpos.bpRegistration;
InterDatanodeProtocol datanode = bpReg.equals(id)?
this: DataNode.createInterDataNodeProtocolProxy(id, getConf(),
- dnConf.socketTimeout);
+ dnConf.socketTimeout, dnConf.connectToDnViaHostname);
ReplicaRecoveryInfo info = callInitReplicaRecovery(datanode, rBlock);
if (info != null &&
info.getGenerationStamp() >= block.getGenerationStamp() &&
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Wed Aug 15 19:04:24 2012
@@ -86,7 +86,7 @@ class DataXceiver extends Receiver imple
private final DataNode datanode;
private final DNConf dnConf;
private final DataXceiverServer dataXceiverServer;
-
+ private final boolean connectToDnViaHostname;
private long opStartTime; //the start time of receiving an Op
private final SocketInputWrapper socketIn;
private OutputStream socketOut;
@@ -113,6 +113,7 @@ class DataXceiver extends Receiver imple
this.isLocal = s.getInetAddress().equals(s.getLocalAddress());
this.datanode = datanode;
this.dataXceiverServer = dataXceiverServer;
+ this.connectToDnViaHostname = datanode.getDnConf().connectToDnViaHostname;
remoteAddress = s.getRemoteSocketAddress().toString();
localAddress = s.getLocalSocketAddress().toString();
@@ -404,7 +405,10 @@ class DataXceiver extends Receiver imple
if (targets.length > 0) {
InetSocketAddress mirrorTarget = null;
// Connect to backup machine
- mirrorNode = targets[0].getXferAddr();
+ mirrorNode = targets[0].getXferAddr(connectToDnViaHostname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + mirrorNode);
+ }
mirrorTarget = NetUtils.createSocketAddr(mirrorNode);
mirrorSock = datanode.newSocket();
try {
@@ -457,7 +461,8 @@ class DataXceiver extends Receiver imple
if (isClient) {
BlockOpResponseProto.newBuilder()
.setStatus(ERROR)
- .setFirstBadLink(mirrorNode)
+ // NB: Unconditionally using the xfer addr w/o hostname
+ .setFirstBadLink(targets[0].getXferAddr())
.build()
.writeDelimitedTo(replyOut);
replyOut.flush();
@@ -729,8 +734,11 @@ class DataXceiver extends Receiver imple
try {
// get the output stream to the proxy
- InetSocketAddress proxyAddr =
- NetUtils.createSocketAddr(proxySource.getXferAddr());
+ final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr);
+ }
+ InetSocketAddress proxyAddr = NetUtils.createSocketAddr(dnAddr);
proxySock = datanode.newSocket();
NetUtils.connect(proxySock, proxyAddr, dnConf.socketTimeout);
proxySock.setSoTimeout(dnConf.socketTimeout);
@@ -891,6 +899,7 @@ class DataXceiver extends Receiver imple
if (mode == BlockTokenSecretManager.AccessMode.WRITE) {
DatanodeRegistration dnR =
datanode.getDNRegistrationForBP(blk.getBlockPoolId());
+ // NB: Unconditionally using the xfer addr w/o hostname
resp.setFirstBadLink(dnR.getXferAddr());
}
resp.build().writeDelimitedTo(out);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Wed Aug 15 19:04:24 2012
@@ -411,7 +411,7 @@ public class DatanodeWebHdfsMethods {
}
final long n = length.getValue() != null? length.getValue()
- : in.getVisibleLength();
+ : in.getVisibleLength() - offset.getValue();
return Response.ok(new OpenEntity(in, n, dfsclient)).type(
MediaType.APPLICATION_OCTET_STREAM).build();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Wed Aug 15 19:04:24 2012
@@ -49,6 +49,8 @@ public class EditLogFileOutputStream ext
private EditsDoubleBuffer doubleBuf;
static ByteBuffer fill = ByteBuffer.allocateDirect(MIN_PREALLOCATION_LENGTH);
+ private static boolean shouldSkipFsyncForTests = false;
+
static {
fill.position(0);
for (int i = 0; i < fill.capacity(); i++) {
@@ -184,7 +186,9 @@ public class EditLogFileOutputStream ext
}
preallocate(); // preallocate file if necessay
doubleBuf.flushTo(fp);
- fc.force(false); // metadata updates not needed
+ if (!shouldSkipFsyncForTests) {
+ fc.force(false); // metadata updates not needed
+ }
}
/**
@@ -247,4 +251,15 @@ public class EditLogFileOutputStream ext
public FileChannel getFileChannelForTesting() {
return fc;
}
+
+ /**
+ * For the purposes of unit tests, we don't need to actually
+ * write durably to disk. So, we can skip the fsync() calls
+ * for a speed improvement.
+ * @param skip true if fsync should <em>not</em> be called
+ */
+ @VisibleForTesting
+ public static void setShouldSkipFsyncForTesting(boolean skip) {
+ shouldSkipFsyncForTests = skip;
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Wed Aug 15 19:04:24 2012
@@ -341,7 +341,7 @@ public class FSEditLog {
* File-based journals are skipped, since they are formatted by the
* Storage format code.
*/
- void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException {
+ synchronized void formatNonFileJournals(NamespaceInfo nsInfo) throws IOException {
Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS,
"Bad state: %s", state);
@@ -352,7 +352,7 @@ public class FSEditLog {
}
}
- List<FormatConfirmable> getFormatConfirmables() {
+ synchronized List<FormatConfirmable> getFormatConfirmables() {
Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS,
"Bad state: %s", state);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Aug 15 19:04:24 2012
@@ -3742,6 +3742,11 @@ public class FSNamesystem implements Nam
this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
this.safeReplication = conf.getInt(DFS_NAMENODE_REPLICATION_MIN_KEY,
DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
+
+ LOG.info(DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY + " = " + threshold);
+ LOG.info(DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY + " = " + datanodeThreshold);
+ LOG.info(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + " = " + extension);
+
// default to safe mode threshold (i.e., don't populate queues before leaving safe mode)
this.replQueueThreshold =
conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY,
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java Wed Aug 15 19:04:24 2012
@@ -127,7 +127,7 @@ public class FileChecksumServlets {
datanode, conf, getUGI(request, conf));
final ClientProtocol nnproxy = dfs.getNamenode();
final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
- path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey());
+ path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey(), false);
MD5MD5CRC32FileChecksum.write(xml, checksum);
} catch(IOException ioe) {
writeXml(ioe, path, xml);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Wed Aug 15 19:04:24 2012
@@ -17,18 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -42,10 +37,12 @@ import org.apache.hadoop.ha.HealthCheckF
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Trash;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.util.ExitUtil.terminate;
+import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -53,9 +50,6 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
-import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -68,8 +62,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -80,14 +72,12 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
+import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.ExitUtil.ExitException;
-
-import static org.apache.hadoop.util.ExitUtil.terminate;
-import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
/**********************************************************
@@ -198,6 +188,22 @@ public class NameNode {
DFS_HA_AUTO_FAILOVER_ENABLED_KEY
};
+ private static final String USAGE = "Usage: java NameNode ["
+ + StartupOption.BACKUP.getName() + "] | ["
+ + StartupOption.CHECKPOINT.getName() + "] | ["
+ + StartupOption.FORMAT.getName() + " ["
+ + StartupOption.CLUSTERID.getName() + " cid ] ["
+ + StartupOption.FORCE.getName() + "] ["
+ + StartupOption.NONINTERACTIVE.getName() + "] ] | ["
+ + StartupOption.UPGRADE.getName() + "] | ["
+ + StartupOption.ROLLBACK.getName() + "] | ["
+ + StartupOption.FINALIZE.getName() + "] | ["
+ + StartupOption.IMPORT.getName() + "] | ["
+ + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | ["
+ + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | ["
+ + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName()
+ + " ] ]";
+
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
if (protocol.equals(ClientProtocol.class.getName())) {
@@ -767,9 +773,18 @@ public class NameNode {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
+
+ if (conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY) == null) {
+ LOG.fatal("No shared edits directory configured for namespace " +
+ nsId + " namenode " + namenodeId);
+ return false;
+ }
+
NNStorage existingStorage = null;
try {
- FSNamesystem fsns = FSNamesystem.loadFromDisk(conf,
+ Configuration confWithoutShared = new Configuration(conf);
+ confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+ FSNamesystem fsns = FSNamesystem.loadFromDisk(confWithoutShared,
FSNamesystem.getNamespaceDirs(conf),
FSNamesystem.getNamespaceEditsDirs(conf, false));
@@ -799,11 +814,9 @@ public class NameNode {
fsns.getFSImage().getEditLog().close();
fsns.getFSImage().getEditLog().initJournalsForWrite();
fsns.getFSImage().getEditLog().recoverUnclosedStreams();
-
- if (copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs,
- newSharedStorage, conf)) {
- return true; // aborted
- }
+
+ copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs, newSharedStorage,
+ conf);
} catch (IOException ioe) {
LOG.error("Could not initialize shared edits dir", ioe);
return true; // aborted
@@ -821,43 +834,59 @@ public class NameNode {
}
return false; // did not abort
}
-
- private static boolean copyEditLogSegmentsToSharedDir(FSNamesystem fsns,
+
+ private static void copyEditLogSegmentsToSharedDir(FSNamesystem fsns,
Collection<URI> sharedEditsDirs, NNStorage newSharedStorage,
- Configuration conf) throws FileNotFoundException, IOException {
+ Configuration conf) throws IOException {
+ Preconditions.checkArgument(!sharedEditsDirs.isEmpty(),
+ "No shared edits specified");
// Copy edit log segments into the new shared edits dir.
- for (JournalAndStream jas : fsns.getFSImage().getEditLog().getJournals()) {
- FileJournalManager fjm = null;
- if (!(jas.getManager() instanceof FileJournalManager)) {
- LOG.error("Cannot populate shared edits dir from non-file " +
- "journal manager: " + jas.getManager());
- return true; // aborted
- } else {
- fjm = (FileJournalManager) jas.getManager();
- }
- for (EditLogFile elf : fjm.getLogFiles(fsns.getFSImage()
- .getMostRecentCheckpointTxId())) {
- File editLogSegment = elf.getFile();
- for (URI sharedEditsUri : sharedEditsDirs) {
- StorageDirectory sharedEditsDir = newSharedStorage
- .getStorageDirectory(sharedEditsUri);
- File targetFile = new File(sharedEditsDir.getCurrentDir(),
- editLogSegment.getName());
- if (!targetFile.exists()) {
- InputStream in = null;
- OutputStream out = null;
- try {
- in = new FileInputStream(editLogSegment);
- out = new AtomicFileOutputStream(targetFile);
- IOUtils.copyBytes(in, out, conf);
- } finally {
- IOUtils.cleanup(LOG, in, out);
- }
- }
+ List<URI> sharedEditsUris = new ArrayList<URI>(sharedEditsDirs);
+ FSEditLog newSharedEditLog = new FSEditLog(conf, newSharedStorage,
+ sharedEditsUris);
+ newSharedEditLog.initJournalsForWrite();
+ newSharedEditLog.recoverUnclosedStreams();
+
+ FSEditLog sourceEditLog = fsns.getFSImage().editLog;
+
+ long fromTxId = fsns.getFSImage().getMostRecentCheckpointTxId();
+ Collection<EditLogInputStream> streams = sourceEditLog.selectInputStreams(
+ fromTxId+1, 0);
+
+ // Set the nextTxid to the CheckpointTxId+1
+ newSharedEditLog.setNextTxId(fromTxId + 1);
+
+ // Copy all edits after last CheckpointTxId to shared edits dir
+ for (EditLogInputStream stream : streams) {
+ LOG.debug("Beginning to copy stream " + stream + " to shared edits");
+ FSEditLogOp op;
+ boolean segmentOpen = false;
+ while ((op = stream.readOp()) != null) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("copying op: " + op);
}
+ if (!segmentOpen) {
+ newSharedEditLog.startLogSegment(op.txid, false);
+ segmentOpen = true;
+ }
+
+ newSharedEditLog.logEdit(op);
+
+ if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) {
+ newSharedEditLog.logSync();
+ newSharedEditLog.endCurrentLogSegment(false);
+ LOG.debug("ending log segment because of END_LOG_SEGMENT op in " + stream);
+ segmentOpen = false;
+ }
+ }
+
+ if (segmentOpen) {
+ LOG.debug("ending log segment because of end of stream in " + stream);
+ newSharedEditLog.logSync();
+ newSharedEditLog.endCurrentLogSegment(false);
+ segmentOpen = false;
}
}
- return false; // did not abort
}
private static boolean finalize(Configuration conf,
@@ -882,25 +911,8 @@ public class NameNode {
return false;
}
- private static void printUsage() {
- System.err.println(
- "Usage: java NameNode [" +
- StartupOption.BACKUP.getName() + "] | [" +
- StartupOption.CHECKPOINT.getName() + "] | [" +
- StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +
- " cid ] [" + StartupOption.FORCE.getName() + "] [" +
- StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
- StartupOption.UPGRADE.getName() + "] | [" +
- StartupOption.ROLLBACK.getName() + "] | [" +
- StartupOption.FINALIZE.getName() + "] | [" +
- StartupOption.IMPORT.getName() + "] | [" +
- StartupOption.INITIALIZESHAREDEDITS.getName() +
- " [" + StartupOption.FORCE.getName() + "] [" +
- StartupOption.NONINTERACTIVE.getName() + "]" +
- "] | [" +
- StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" +
- StartupOption.RECOVER.getName() + " [ " +
- StartupOption.FORCE.getName() + " ] ]");
+ private static void printUsage(PrintStream out) {
+ out.println(USAGE + "\n");
}
private static StartupOption parseArguments(String args[]) {
@@ -1048,7 +1060,7 @@ public class NameNode {
conf = new HdfsConfiguration();
StartupOption startOpt = parseArguments(argv);
if (startOpt == null) {
- printUsage();
+ printUsage(System.err);
return null;
}
setStartupOption(conf, startOpt);
@@ -1162,6 +1174,10 @@ public class NameNode {
/**
*/
public static void main(String argv[]) throws Exception {
+ if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
try {
StringUtils.startupShutdownMessage(NameNode.class, argv, LOG);
NameNode namenode = createNameNode(argv, null);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Wed Aug 15 19:04:24 2012
@@ -562,6 +562,9 @@ public class SecondaryNameNode implement
if (opts == null) {
LOG.fatal("Failed to parse options");
terminate(1);
+ } else if (opts.shouldPrintHelp()) {
+ opts.usage();
+ System.exit(0);
}
StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
@@ -595,6 +598,7 @@ public class SecondaryNameNode implement
private final Option geteditsizeOpt;
private final Option checkpointOpt;
private final Option formatOpt;
+ private final Option helpOpt;
Command cmd;
@@ -605,6 +609,7 @@ public class SecondaryNameNode implement
private boolean shouldForce;
private boolean shouldFormat;
+ private boolean shouldPrintHelp;
CommandLineOpts() {
geteditsizeOpt = new Option("geteditsize",
@@ -612,20 +617,32 @@ public class SecondaryNameNode implement
checkpointOpt = OptionBuilder.withArgName("force")
.hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");;
formatOpt = new Option("format", "format the local storage during startup");
+ helpOpt = new Option("h", "help", false, "get help information");
options.addOption(geteditsizeOpt);
options.addOption(checkpointOpt);
options.addOption(formatOpt);
+ options.addOption(helpOpt);
}
public boolean shouldFormat() {
return shouldFormat;
}
+ public boolean shouldPrintHelp() {
+ return shouldPrintHelp;
+ }
+
public void parse(String ... argv) throws ParseException {
CommandLineParser parser = new PosixParser();
CommandLine cmdLine = parser.parse(options, argv);
+ if (cmdLine.hasOption(helpOpt.getOpt())
+ || cmdLine.hasOption(helpOpt.getLongOpt())) {
+ shouldPrintHelp = true;
+ return;
+ }
+
boolean hasGetEdit = cmdLine.hasOption(geteditsizeOpt.getOpt());
boolean hasCheckpoint = cmdLine.hasOption(checkpointOpt.getOpt());
if (hasGetEdit && hasCheckpoint) {
@@ -662,8 +679,13 @@ public class SecondaryNameNode implement
}
void usage() {
+ String header = "The Secondary NameNode is a helper "
+ + "to the primary NameNode. The Secondary is responsible "
+ + "for supporting periodic checkpoints of the HDFS metadata. "
+ + "The current design allows only one Secondary NameNode "
+ + "per HDFS cluster.";
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("secondarynamenode", options);
+ formatter.printHelp("secondarynamenode", header, options, "", false);
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java Wed Aug 15 19:04:24 2012
@@ -42,6 +42,10 @@ public class DFSHAAdmin extends HAAdmin
protected void setErrOut(PrintStream errOut) {
this.errOut = errOut;
}
+
+ protected void setOut(PrintStream out) {
+ this.out = out;
+ }
@Override
public void setConf(Configuration conf) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java?rev=1373573&r1=1373572&r2=1373573&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java Wed Aug 15 19:04:24 2012
@@ -162,6 +162,10 @@ public class DFSZKFailoverController ext
public static void main(String args[])
throws Exception {
+ if (DFSUtil.parseHelpArgument(args,
+ ZKFailoverController.USAGE, System.out, true)) {
+ System.exit(0);
+ }
GenericOptionsParser parser = new GenericOptionsParser(
new HdfsConfiguration(), args);