You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/03/11 18:56:07 UTC
svn commit: r1299412 [9/11] - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/ src/main/bin/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/...
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java Sun Mar 11 17:55:58 2012
@@ -302,7 +302,7 @@ public class TestDataTransferProtocol ex
testWrite(firstBlock, BlockConstructionStage.PIPELINE_SETUP_CREATE, 0L,
"Cannot create a RBW block", true);
// test PIPELINE_SETUP_APPEND on an existing block
- newGS = newBlock.getGenerationStamp() + 1;
+ newGS = firstBlock.getGenerationStamp() + 1;
testWrite(firstBlock, BlockConstructionStage.PIPELINE_SETUP_APPEND,
newGS, "Cannot append to a RBW replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java Sun Mar 11 17:55:58 2012
@@ -279,7 +279,8 @@ public class TestDecommission {
* @throws IOException */
private void startCluster(int numNameNodes, int numDatanodes,
Configuration conf) throws IOException {
- cluster = new MiniDFSCluster.Builder(conf).numNameNodes(numNameNodes)
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(numNameNodes))
.numDataNodes(numDatanodes).build();
cluster.waitActive();
for (int i = 0; i < numNameNodes; i++) {
@@ -507,7 +508,8 @@ public class TestDecommission {
InterruptedException {
conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath());
int numDatanodes = 1;
- cluster = new MiniDFSCluster.Builder(conf).numNameNodes(numNameNodes)
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(numNameNodes))
.numDataNodes(numDatanodes).setupHostsFile(true).build();
cluster.waitActive();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java Sun Mar 11 17:55:58 2012
@@ -90,7 +90,7 @@ public class TestFileAppendRestart {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
FileSystem fs = cluster.getFileSystem();
File editLog =
- new File(FSImageTestUtil.getNameNodeCurrentDirs(cluster).get(0),
+ new File(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(0),
NNStorage.getInProgressEditsFileName(1));
EnumMap<FSEditLogOpCodes, Holder<Integer>> counts;
@@ -98,18 +98,31 @@ public class TestFileAppendRestart {
writeAndAppend(fs, p1, BLOCK_SIZE, BLOCK_SIZE);
counts = FSImageTestUtil.countEditLogOpTypes(editLog);
+ // OP_ADD to create file
+ // OP_UPDATE_BLOCKS for first block
+ // OP_CLOSE to close file
+ // OP_ADD to reopen file
+ // OP_UPDATE_BLOCKS for second block
+ // OP_CLOSE to close file
assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+ assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
Path p2 = new Path("/not-block-boundaries");
writeAndAppend(fs, p2, BLOCK_SIZE/2, BLOCK_SIZE);
counts = FSImageTestUtil.countEditLogOpTypes(editLog);
- // We get *3* OP_ADDS from this test rather than two. The first
- // OP_ADD comes from re-opening the file to establish the lease,
- // the second comes from the updatePipeline call when the block
- // itself has its generation stamp incremented
- assertEquals(5, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
- assertEquals(4, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
+ // OP_ADD to create file
+ // OP_UPDATE_BLOCKS for first block
+ // OP_CLOSE to close file
+ // OP_ADD to re-establish the lease
+ // OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block)
+ // OP_UPDATE_BLOCKS at the start of the second block
+ // OP_CLOSE to close file
+ // Total: 2 OP_ADDs, 3 OP_UPDATE_BLOCKS, and 2 OP_CLOSEs in addition
+ // to the ones above
+ assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+ assertEquals(2+3, (int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
+ assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
cluster.restartNameNode();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java Sun Mar 11 17:55:58 2012
@@ -146,8 +146,14 @@ public class TestFileCorruption extends
// report corrupted block by the third datanode
DatanodeRegistration dnR =
DataNodeTestUtils.getDNRegistrationForBP(dataNode, blk.getBlockPoolId());
- cluster.getNamesystem().getBlockManager().findAndMarkBlockAsCorrupt(
- blk, new DatanodeInfo(dnR), "TEST");
+ FSNamesystem ns = cluster.getNamesystem();
+ ns.writeLock();
+ try {
+ cluster.getNamesystem().getBlockManager().findAndMarkBlockAsCorrupt(
+ blk, new DatanodeInfo(dnR), "TEST");
+ } finally {
+ ns.writeUnlock();
+ }
// open the file
fs.open(FILE_PATH);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java Sun Mar 11 17:55:58 2012
@@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
@@ -34,8 +33,6 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.security.UserGroupInformation;
-
import junit.framework.TestCase;
/**
* This class tests if block replacement request to data nodes work correctly.
@@ -97,8 +94,8 @@ public class TestGetBlocks extends TestC
// get RPC client to namenode
InetSocketAddress addr = new InetSocketAddress("localhost",
cluster.getNameNodePort());
- NamenodeProtocol namenode = new NamenodeProtocolTranslatorPB(addr, CONF,
- UserGroupInformation.getCurrentUser());
+ NamenodeProtocol namenode = NameNodeProxies.createProxy(CONF,
+ NameNode.getUri(addr), NamenodeProtocol.class).getProxy();
// get blocks of size fileLen from dataNodes[0]
BlockWithLocations[] locs;
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java Sun Mar 11 17:55:58 2012
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import junit.framework.Assert;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
@@ -31,8 +32,13 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
+import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,7 +55,7 @@ public class TestIsMethodSupported {
@BeforeClass
public static void setUp() throws Exception {
- cluster = (new MiniDFSCluster.Builder(conf)).numNameNodes(1)
+ cluster = (new MiniDFSCluster.Builder(conf))
.numDataNodes(1).build();
nnAddress = cluster.getNameNode().getNameNodeAddress();
dnAddress = new InetSocketAddress(cluster.getDataNodes().get(0)
@@ -66,8 +72,9 @@ public class TestIsMethodSupported {
@Test
public void testNamenodeProtocol() throws IOException {
NamenodeProtocolTranslatorPB translator =
- new NamenodeProtocolTranslatorPB(nnAddress, conf,
- UserGroupInformation.getCurrentUser());
+ (NamenodeProtocolTranslatorPB) NameNodeProxies.createNonHAProxy(conf,
+ nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
+ true).getProxy();
boolean exists = translator.isMethodSupported("rollEditLog");
Assert.assertTrue(exists);
exists = translator.isMethodSupported("bogusMethod");
@@ -99,15 +106,17 @@ public class TestIsMethodSupported {
@Test
public void testClientNamenodeProtocol() throws IOException {
ClientNamenodeProtocolTranslatorPB translator =
- new ClientNamenodeProtocolTranslatorPB(nnAddress, conf,
- UserGroupInformation.getCurrentUser());
+ (ClientNamenodeProtocolTranslatorPB) NameNodeProxies.createNonHAProxy(
+ conf, nnAddress, ClientProtocol.class,
+ UserGroupInformation.getCurrentUser(), true).getProxy();
Assert.assertTrue(translator.isMethodSupported("mkdirs"));
}
@Test
public void tesJournalProtocol() throws IOException {
- JournalProtocolTranslatorPB translator =
- new JournalProtocolTranslatorPB(nnAddress, conf);
+ JournalProtocolTranslatorPB translator = (JournalProtocolTranslatorPB)
+ NameNodeProxies.createNonHAProxy(conf, nnAddress, JournalProtocol.class,
+ UserGroupInformation.getCurrentUser(), true).getProxy();
//Nameode doesn't implement JournalProtocol
Assert.assertFalse(translator.isMethodSupported("startLogSegment"));
}
@@ -130,24 +139,30 @@ public class TestIsMethodSupported {
@Test
public void testGetUserMappingsProtocol() throws IOException {
GetUserMappingsProtocolClientSideTranslatorPB translator =
- new GetUserMappingsProtocolClientSideTranslatorPB(
- nnAddress, UserGroupInformation.getCurrentUser(), conf);
+ (GetUserMappingsProtocolClientSideTranslatorPB)
+ NameNodeProxies.createNonHAProxy(conf, nnAddress,
+ GetUserMappingsProtocol.class, UserGroupInformation.getCurrentUser(),
+ true).getProxy();
Assert.assertTrue(translator.isMethodSupported("getGroupsForUser"));
}
@Test
public void testRefreshAuthorizationPolicyProtocol() throws IOException {
- RefreshAuthorizationPolicyProtocolClientSideTranslatorPB translator =
- new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(
- nnAddress, UserGroupInformation.getCurrentUser(), conf);
+ RefreshAuthorizationPolicyProtocolClientSideTranslatorPB translator =
+ (RefreshAuthorizationPolicyProtocolClientSideTranslatorPB)
+ NameNodeProxies.createNonHAProxy(conf, nnAddress,
+ RefreshAuthorizationPolicyProtocol.class,
+ UserGroupInformation.getCurrentUser(), true).getProxy();
Assert.assertTrue(translator.isMethodSupported("refreshServiceAcl"));
}
@Test
public void testRefreshUserMappingsProtocol() throws IOException {
RefreshUserMappingsProtocolClientSideTranslatorPB translator =
- new RefreshUserMappingsProtocolClientSideTranslatorPB(
- nnAddress, UserGroupInformation.getCurrentUser(), conf);
+ (RefreshUserMappingsProtocolClientSideTranslatorPB)
+ NameNodeProxies.createNonHAProxy(conf, nnAddress,
+ RefreshUserMappingsProtocol.class,
+ UserGroupInformation.getCurrentUser(), true).getProxy();
Assert.assertTrue(
translator.isMethodSupported("refreshUserToGroupsMappings"));
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java Sun Mar 11 17:55:58 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -37,11 +38,13 @@ public class TestMiniDFSCluster {
private static final String CLUSTER_1 = "cluster1";
private static final String CLUSTER_2 = "cluster2";
private static final String CLUSTER_3 = "cluster3";
+ private static final String CLUSTER_4 = "cluster4";
protected String testDataPath;
protected File testDataDir;
@Before
public void setUp() {
- testDataPath = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
+ testDataPath = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA,
+ "build/test/data");
testDataDir = new File(new File(testDataPath).getParentFile(),
"miniclusters");
@@ -103,5 +106,21 @@ public class TestMiniDFSCluster {
}
}
-
+ @Test(timeout=100000)
+ public void testIsClusterUpAfterShutdown() throws Throwable {
+ Configuration conf = new HdfsConfiguration();
+ File testDataCluster4 = new File(testDataPath, CLUSTER_4);
+ String c4Path = testDataCluster4.getAbsolutePath();
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c4Path);
+ MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build();
+ try {
+ DistributedFileSystem dfs = (DistributedFileSystem) cluster4.getFileSystem();
+ dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_ENTER);
+ cluster4.shutdown();
+ } finally {
+ while(cluster4.isClusterUp()){
+ Thread.sleep(1000);
+ }
+ }
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java Sun Mar 11 17:55:58 2012
@@ -75,7 +75,8 @@ public class TestReplication extends Tes
private void checkFile(FileSystem fileSys, Path name, int repl)
throws IOException {
Configuration conf = fileSys.getConf();
- ClientProtocol namenode = DFSUtil.createNamenode(conf);
+ ClientProtocol namenode = NameNodeProxies.createProxy(conf, fileSys.getUri(),
+ ClientProtocol.class).getProxy();
waitForBlockReplication(name.toString(), namenode,
Math.min(numDatanodes, repl), -1);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java Sun Mar 11 17:55:58 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.security;
+import static org.junit.Assert.*;
+
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
@@ -32,12 +34,16 @@ import org.apache.commons.logging.LogFac
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
@@ -64,6 +70,7 @@ public class TestDelegationToken {
config.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
+ config.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
config.set("hadoop.security.auth_to_local",
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0");
@@ -71,7 +78,6 @@ public class TestDelegationToken {
cluster.waitActive();
dtSecretManager = NameNodeAdapter.getDtSecretManager(
cluster.getNamesystem());
- dtSecretManager.startThreads();
}
@After
@@ -269,5 +275,51 @@ public class TestDelegationToken {
}
});
}
-
+
+ /**
+ * Test that the delegation token secret manager only runs when the
+ * NN is out of safe mode. This is because the secret manager
+ * has to log to the edit log, which should not be written in
+ * safe mode. Regression test for HDFS-2579.
+ */
+ @Test
+ public void testDTManagerInSafeMode() throws Exception {
+ cluster.startDataNodes(config, 1, true, StartupOption.REGULAR, null);
+ FileSystem fs = cluster.getFileSystem();
+ for (int i = 0; i < 5; i++) {
+ DFSTestUtil.createFile(fs, new Path("/test-" + i), 100, (short)1, 1L);
+ }
+ cluster.getConfiguration(0).setInt(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY, 500);
+ cluster.getConfiguration(0).setInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 30000);
+ cluster.setWaitSafeMode(false);
+ cluster.restartNameNode();
+ NameNode nn = cluster.getNameNode();
+ assertTrue(nn.isInSafeMode());
+ DelegationTokenSecretManager sm =
+ NameNodeAdapter.getDtSecretManager(nn.getNamesystem());
+ assertFalse("Secret manager should not run in safe mode", sm.isRunning());
+
+ NameNodeAdapter.leaveSafeMode(nn, false);
+ assertTrue("Secret manager should start when safe mode is exited",
+ sm.isRunning());
+
+ LOG.info("========= entering safemode again");
+
+ NameNodeAdapter.enterSafeMode(nn, false);
+ assertFalse("Secret manager should stop again when safe mode " +
+ "is manually entered", sm.isRunning());
+
+ // Set the cluster to leave safemode quickly on its own.
+ cluster.getConfiguration(0).setInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
+ cluster.setWaitSafeMode(true);
+ cluster.restartNameNode();
+ nn = cluster.getNameNode();
+ sm = NameNodeAdapter.getDtSecretManager(nn.getNamesystem());
+
+ assertFalse(nn.isInSafeMode());
+ assertTrue(sm.isRunning());
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java Sun Mar 11 17:55:58 2012
@@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.Distribute
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
@@ -114,11 +113,12 @@ public class TestDelegationTokenForProxy
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
config.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER),
"group1");
+ config.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
configureSuperUserIPAddresses(config, REAL_USER);
FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0");
cluster = new MiniDFSCluster.Builder(config).build();
cluster.waitActive();
- NameNodeAdapter.getDtSecretManager(cluster.getNamesystem()).startThreads();
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Sun Mar 11 17:55:58 2012
@@ -373,7 +373,7 @@ public class TestBlockToken {
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(1)
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1).build();
cluster.waitActive();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Sun Mar 11 17:55:58 2012
@@ -18,9 +18,10 @@
package org.apache.hadoop.hdfs.server.balancer;
import java.io.IOException;
-import java.net.InetSocketAddress;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeoutException;
@@ -37,28 +38,28 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
/**
* This class tests if a balancer schedules tasks correctly.
*/
public class TestBalancer extends TestCase {
private static final Log LOG = LogFactory.getLog(
- "org.apache.hadoop.hdfs.TestReplication");
+ "org.apache.hadoop.hdfs.TestBalancer");
- final private static long CAPACITY = 500L;
- final private static String RACK0 = "/rack0";
- final private static String RACK1 = "/rack1";
- final private static String RACK2 = "/rack2";
- final static private String fileName = "/tmp.txt";
- final static private Path filePath = new Path(fileName);
+ final static long CAPACITY = 500L;
+ final static String RACK0 = "/rack0";
+ final static String RACK1 = "/rack1";
+ final static String RACK2 = "/rack2";
+ final private static String fileName = "/tmp.txt";
+ final static Path filePath = new Path(fileName);
private MiniDFSCluster cluster;
ClientProtocol client;
@@ -82,9 +83,10 @@ public class TestBalancer extends TestCa
}
/* create a file with a length of <code>fileLen</code> */
- private void createFile(long fileLen, short replicationFactor)
+ static void createFile(MiniDFSCluster cluster, Path filePath, long fileLen,
+ short replicationFactor, int nnIndex)
throws IOException {
- FileSystem fs = cluster.getFileSystem();
+ FileSystem fs = cluster.getFileSystem(nnIndex);
DFSTestUtil.createFile(fs, filePath, fileLen,
replicationFactor, r.nextLong());
DFSTestUtil.waitReplication(fs, filePath, replicationFactor);
@@ -99,11 +101,12 @@ public class TestBalancer extends TestCa
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numNodes).build();
try {
cluster.waitActive();
- client = DFSUtil.createNamenode(conf);
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
short replicationFactor = (short)(numNodes-1);
long fileLen = size/replicationFactor;
- createFile(fileLen, replicationFactor);
+ createFile(cluster , filePath, fileLen, replicationFactor, 0);
List<LocatedBlock> locatedBlocks = client.
getBlockLocations(fileName, 0, fileLen).getLocatedBlocks();
@@ -193,7 +196,8 @@ public class TestBalancer extends TestCa
.simulatedCapacities(capacities)
.build();
cluster.waitActive();
- client = DFSUtil.createNamenode(conf);
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
for(int i = 0; i < blocksDN.length; i++)
cluster.injectBlocks(i, Arrays.asList(blocksDN[i]));
@@ -211,7 +215,8 @@ public class TestBalancer extends TestCa
* @throws IOException - if getStats() fails
* @throws TimeoutException
*/
- private void waitForHeartBeat(long expectedUsedSpace, long expectedTotalSpace)
+ static void waitForHeartBeat(long expectedUsedSpace,
+ long expectedTotalSpace, ClientProtocol client, MiniDFSCluster cluster)
throws IOException, TimeoutException {
long timeout = TIMEOUT;
long failtime = (timeout <= 0L) ? Long.MAX_VALUE
@@ -248,7 +253,8 @@ public class TestBalancer extends TestCa
* @throws IOException
* @throws TimeoutException
*/
- private void waitForBalancer(long totalUsedSpace, long totalCapacity)
+ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
+ ClientProtocol client, MiniDFSCluster cluster)
throws IOException, TimeoutException {
long timeout = TIMEOUT;
long failtime = (timeout <= 0L) ? Long.MAX_VALUE
@@ -305,13 +311,15 @@ public class TestBalancer extends TestCa
.build();
try {
cluster.waitActive();
- client = DFSUtil.createNamenode(conf);
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
long totalCapacity = sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity*3/10;
- createFile(totalUsedSpace/numOfDatanodes, (short)numOfDatanodes);
+ createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
+ (short) numOfDatanodes, 0);
// start up an empty node with the same capacity and on the same rack
cluster.startDataNodes(conf, 1, true, null,
new String[]{newRack}, new long[]{newCapacity});
@@ -327,17 +335,16 @@ public class TestBalancer extends TestCa
private void runBalancer(Configuration conf,
long totalUsedSpace, long totalCapacity) throws Exception {
- waitForHeartBeat(totalUsedSpace, totalCapacity);
+ waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
// start rebalancing
- final List<InetSocketAddress> namenodes =new ArrayList<InetSocketAddress>();
- namenodes.add(NameNode.getServiceAddress(conf, true));
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
- waitForHeartBeat(totalUsedSpace, totalCapacity);
+ waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
LOG.info("Rebalancing with default ctor.");
- waitForBalancer(totalUsedSpace, totalCapacity);
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
}
/** one-node cluster test*/
@@ -396,13 +403,15 @@ public class TestBalancer extends TestCa
.build();
try {
cluster.waitActive();
- client = DFSUtil.createNamenode(conf);
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
long totalCapacity = sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
- createFile(totalUsedSpace / numOfDatanodes, (short) numOfDatanodes);
+ createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
+ (short) numOfDatanodes, 0);
// start up an empty node with the same capacity and on the same rack
cluster.startDataNodes(conf, 1, true, null, new String[] { newRack },
new long[] { newCapacity });
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Sun Mar 11 17:55:58 2012
@@ -18,8 +18,9 @@
package org.apache.hadoop.hdfs.server.balancer;
import java.io.IOException;
-import java.net.InetSocketAddress;
+import java.net.URI;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Random;
@@ -34,12 +35,13 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -155,7 +157,7 @@ public class TestBalancerWithMultipleNam
LOG.info("BALANCER 1");
// start rebalancing
- final List<InetSocketAddress> namenodes = DFSUtil.getNNServiceRpcAddresses(s.conf);
+ final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(s.conf);
final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, s.conf);
Assert.assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
@@ -249,8 +251,9 @@ public class TestBalancerWithMultipleNam
final ExtendedBlock[][] blocks;
{
LOG.info("UNEVEN 1");
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numNameNodes(nNameNodes)
+ final MiniDFSCluster cluster = new MiniDFSCluster
+ .Builder(new Configuration(conf))
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
.numDataNodes(nDataNodes)
.racks(racks)
.simulatedCapacities(capacities)
@@ -258,6 +261,7 @@ public class TestBalancerWithMultipleNam
LOG.info("UNEVEN 2");
try {
cluster.waitActive();
+ DFSTestUtil.setFederatedConfiguration(cluster, conf);
LOG.info("UNEVEN 3");
final Suite s = new Suite(cluster, nNameNodes, nDataNodes, conf);
blocks = generateBlocks(s, usedSpacePerNN);
@@ -271,7 +275,7 @@ public class TestBalancerWithMultipleNam
{
LOG.info("UNEVEN 10");
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numNameNodes(nNameNodes)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
.numDataNodes(nDataNodes)
.racks(racks)
.simulatedCapacities(capacities)
@@ -324,13 +328,15 @@ public class TestBalancerWithMultipleNam
Assert.assertEquals(nDataNodes, racks.length);
LOG.info("RUN_TEST -1");
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numNameNodes(nNameNodes)
+ final MiniDFSCluster cluster = new MiniDFSCluster
+ .Builder(new Configuration(conf))
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
.numDataNodes(nDataNodes)
.racks(racks)
.simulatedCapacities(capacities)
.build();
LOG.info("RUN_TEST 0");
+ DFSTestUtil.setFederatedConfiguration(cluster, conf);
try {
cluster.waitActive();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Sun Mar 11 17:55:58 2012
@@ -24,8 +24,11 @@ import java.util.Iterator;
import java.util.Set;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.util.Daemon;
+import org.junit.Assert;
import com.google.common.base.Preconditions;
@@ -124,6 +127,58 @@ public class BlockManagerTestUtil {
return blockManager.computeDatanodeWork();
}
+ public static int computeInvalidationWork(BlockManager bm) {
+ return bm.computeInvalidateWork(Integer.MAX_VALUE);
+ }
+
+ /**
+ * Compute all the replication and invalidation work for the
+ * given BlockManager.
+ *
+ * This differs from the above functions in that it computes
+ * replication work for all DNs rather than a particular subset,
+ * regardless of invalidation/replication limit configurations.
+ *
+ * NB: you may want to set
+ * {@link DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY} to
+ * a high value to ensure that all work is calculated.
+ */
+ public static int computeAllPendingWork(BlockManager bm)
+ throws IOException {
+ int work = computeInvalidationWork(bm);
+ work += bm.computeReplicationWork(Integer.MAX_VALUE);
+ return work;
+ }
+
+ /**
+ * Ensure that the given NameNode marks the specified DataNode as
+ * entirely dead/expired.
+ * @param nn the NameNode to manipulate
+ * @param dnName the name of the DataNode
+ */
+ public static void noticeDeadDatanode(NameNode nn, String dnName) {
+ FSNamesystem namesystem = nn.getNamesystem();
+ namesystem.writeLock();
+ try {
+ DatanodeManager dnm = namesystem.getBlockManager().getDatanodeManager();
+ HeartbeatManager hbm = dnm.getHeartbeatManager();
+ DatanodeDescriptor[] dnds = hbm.getDatanodes();
+ DatanodeDescriptor theDND = null;
+ for (DatanodeDescriptor dnd : dnds) {
+ if (dnd.getName().equals(dnName)) {
+ theDND = dnd;
+ }
+ }
+ Assert.assertNotNull("Could not find DN with name: " + dnName, theDND);
+
+ synchronized (hbm) {
+ theDND.setLastUpdate(0);
+ hbm.heartbeatCheck();
+ }
+ } finally {
+ namesystem.writeUnlock();
+ }
+ }
/**
* Change whether the block placement policy will prefer the writer's
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Sun Mar 11 17:55:58 2012
@@ -41,7 +41,7 @@ import org.apache.hadoop.hdfs.server.pro
public class TestHeartbeatHandling extends TestCase {
/**
* Test if
- * {@link FSNamesystem#handleHeartbeat(DatanodeRegistration, long, long, long, long, int, int)}
+ * {@link FSNamesystem#handleHeartbeat}
* can pick up replication and/or invalidate requests and observes the max
* limit
*/
@@ -75,7 +75,8 @@ public class TestHeartbeatHandling exten
dd.addBlockToBeReplicated(
new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP), ONE_TARGET);
}
- DatanodeCommand[]cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem);
+ DatanodeCommand[] cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd,
+ namesystem).getCommands();
assertEquals(1, cmds.length);
assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
@@ -85,27 +86,31 @@ public class TestHeartbeatHandling exten
blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
}
dd.addBlocksToBeInvalidated(blockList);
- cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem);
+ cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem)
+ .getCommands();
assertEquals(2, cmds.length);
assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
- cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem);
+ cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem)
+ .getCommands();
assertEquals(2, cmds.length);
assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
- cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem);
+ cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem)
+ .getCommands();
assertEquals(1, cmds.length);
assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[0].getAction());
assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
- cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(null, cmds);
+ cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem)
+ .getCommands();
+ assertEquals(0, cmds.length);
}
} finally {
namesystem.writeUnlock();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java Sun Mar 11 17:55:58 2012
@@ -81,15 +81,8 @@ public class TestNodeCount extends TestC
DataNodeProperties dnprop = cluster.stopDataNode(datanode.getName());
// make sure that NN detects that the datanode is down
- try {
- namesystem.writeLock();
- synchronized (hm) {
- datanode.setLastUpdate(0); // mark it dead
- hm.heartbeatCheck();
- }
- } finally {
- namesystem.writeUnlock();
- }
+ BlockManagerTestUtil.noticeDeadDatanode(
+ cluster.getNameNode(), datanode.getName());
// the block will be replicated
DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
@@ -121,16 +114,8 @@ public class TestNodeCount extends TestC
// bring down non excessive datanode
dnprop = cluster.stopDataNode(nonExcessDN.getName());
// make sure that NN detects that the datanode is down
-
- try {
- namesystem.writeLock();
- synchronized(hm) {
- nonExcessDN.setLastUpdate(0); // mark it dead
- hm.heartbeatCheck();
- }
- } finally {
- namesystem.writeUnlock();
- }
+ BlockManagerTestUtil.noticeDeadDatanode(
+ cluster.getNameNode(), nonExcessDN.getName());
// The block should be replicated
initializeTimeout(TIMEOUT);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java Sun Mar 11 17:55:58 2012
@@ -50,6 +50,29 @@ public class DataNodeAdapter {
boolean heartbeatsDisabledForTests) {
dn.setHeartbeatsDisabledForTests(heartbeatsDisabledForTests);
}
+
+ public static void triggerDeletionReport(DataNode dn) throws IOException {
+ for (BPOfferService bpos : dn.getAllBpOs()) {
+ bpos.triggerDeletionReportForTests();
+ }
+ }
+
+ public static void triggerHeartbeat(DataNode dn) throws IOException {
+ for (BPOfferService bpos : dn.getAllBpOs()) {
+ bpos.triggerHeartbeatForTests();
+ }
+ }
+
+ public static void triggerBlockReport(DataNode dn) throws IOException {
+ for (BPOfferService bpos : dn.getAllBpOs()) {
+ bpos.triggerBlockReportForTests();
+ }
+ }
+
+ public static long getPendingAsyncDeletions(DataNode dn) {
+ FSDataset fsd = (FSDataset)dn.getFSDataset();
+ return fsd.asyncDiskService.countPendingDeletions();
+ }
/**
* Insert a Mockito spy object between the given DataNode and
@@ -69,11 +92,20 @@ public class DataNodeAdapter {
}
Preconditions.checkArgument(bpos != null,
"No such bpid: %s", bpid);
+
+ BPServiceActor bpsa = null;
+ for (BPServiceActor thisBpsa : bpos.getBPServiceActors()) {
+ if (thisBpsa.getNNSocketAddress().equals(nn.getServiceRpcAddress())) {
+ bpsa = thisBpsa;
+ break;
+ }
+ }
+ Preconditions.checkArgument(bpsa != null,
+ "No service actor to NN at %s", nn.getServiceRpcAddress());
- DatanodeProtocolClientSideTranslatorPB origNN = bpos.getBpNamenode();
+ DatanodeProtocolClientSideTranslatorPB origNN = bpsa.getNameNodeProxy();
DatanodeProtocolClientSideTranslatorPB spy = Mockito.spy(origNN);
-
- bpos.setBpNamenode(spy);
+ bpsa.setNameNode(spy);
return spy;
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Sun Mar 11 17:55:58 2012
@@ -42,10 +42,13 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
+import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Daemon;
@@ -137,7 +140,9 @@ public class TestBlockRecovery {
Mockito.anyInt(),
Mockito.anyInt(),
Mockito.anyInt()))
- .thenReturn(new DatanodeCommand[0]);
+ .thenReturn(new HeartbeatResponse(
+ new DatanodeCommand[0],
+ new NNHAStatusHeartbeat(State.ACTIVE, 1)));
dn = new DataNode(conf, dirs, null) {
@Override
@@ -147,14 +152,8 @@ public class TestBlockRecovery {
return namenode;
}
};
- dn.runDatanodeDaemon();
- while (!dn.isDatanodeFullyStarted()) {
- try {
- Thread.sleep(50);
- } catch (InterruptedException e) {
- fail("Interrupted starting DN");
- }
- }
+ // Trigger a heartbeat so that it acknowledges the NN as active.
+ dn.getAllBpOs()[0].triggerHeartbeatForTests();
}
/**
@@ -462,7 +461,7 @@ public class TestBlockRecovery {
initReplicaRecovery(any(RecoveringBlock.class));
Daemon d = spyDN.recoverBlocks(initRecoveringBlocks());
d.join();
- DatanodeProtocol dnP = dn.getBPNamenode(POOL_ID);
+ DatanodeProtocol dnP = dn.getActiveNamenodeForBP(POOL_ID);
verify(dnP).commitBlockSynchronization(
block, RECOVERY_ID, 0, true, true, DatanodeID.EMPTY_ARRAY);
}
@@ -519,7 +518,7 @@ public class TestBlockRecovery {
} catch (IOException e) {
e.getMessage().startsWith("Cannot recover ");
}
- DatanodeProtocol namenode = dn.getBPNamenode(POOL_ID);
+ DatanodeProtocol namenode = dn.getActiveNamenodeForBP(POOL_ID);
verify(namenode, never()).commitBlockSynchronization(
any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(),
anyBoolean(), any(DatanodeID[].class));
@@ -548,7 +547,7 @@ public class TestBlockRecovery {
} catch (IOException e) {
e.getMessage().startsWith("Cannot recover ");
}
- DatanodeProtocol namenode = dn.getBPNamenode(POOL_ID);
+ DatanodeProtocol namenode = dn.getActiveNamenodeForBP(POOL_ID);
verify(namenode, never()).commitBlockSynchronization(
any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(),
anyBoolean(), any(DatanodeID[].class));
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java Sun Mar 11 17:55:58 2012
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -36,7 +37,6 @@ import org.junit.Test;
* Tests if DataNode process exits if all Block Pool services exit.
*/
public class TestDataNodeExit {
- private static int BASEPORT = 9923;
private static long WAIT_TIME_IN_MILLIS = 10;
Configuration conf;
MiniDFSCluster cluster = null;
@@ -46,8 +46,9 @@ public class TestDataNodeExit {
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 100);
- cluster = new MiniDFSCluster.Builder(conf).numNameNodes(3)
- .nameNodePort(BASEPORT).build();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(3))
+ .build();
for (int i = 0; i < 3; i++) {
cluster.waitActive(i);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Sun Mar 11 17:55:58 2012
@@ -23,6 +23,8 @@ import static org.junit.Assert.assertNot
import static org.junit.Assert.assertNotSame;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -30,6 +32,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -55,8 +58,9 @@ public class TestDataNodeMultipleRegistr
*/
@Test
public void test2NNRegistration() throws IOException {
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(2)
- .nameNodePort(9928).build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .build();
try {
cluster.waitActive();
NameNode nn1 = cluster.getNameNode(0);
@@ -90,23 +94,22 @@ public class TestDataNodeMultipleRegistr
assertEquals("number of volumes is wrong", 2, volInfos.size());
for (BPOfferService bpos : dn.getAllBpOs()) {
- LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration.name + "; sid="
- + bpos.bpRegistration.storageID + "; nna=" + bpos.nnAddr);
+ LOG.info("BP: " + bpos);
}
BPOfferService bpos1 = dn.getAllBpOs()[0];
BPOfferService bpos2 = dn.getAllBpOs()[1];
// The order of bpos is not guaranteed, so fix the order
- if (bpos1.nnAddr.equals(nn2.getNameNodeAddress())) {
+ if (getNNSocketAddress(bpos1).equals(nn2.getNameNodeAddress())) {
BPOfferService tmp = bpos1;
bpos1 = bpos2;
bpos2 = tmp;
}
- assertEquals("wrong nn address", bpos1.nnAddr,
+ assertEquals("wrong nn address", getNNSocketAddress(bpos1),
nn1.getNameNodeAddress());
- assertEquals("wrong nn address", bpos2.nnAddr,
+ assertEquals("wrong nn address", getNNSocketAddress(bpos2),
nn2.getNameNodeAddress());
assertEquals("wrong bpid", bpos1.getBlockPoolId(), bpid1);
assertEquals("wrong bpid", bpos2.getBlockPoolId(), bpid2);
@@ -120,6 +123,12 @@ public class TestDataNodeMultipleRegistr
cluster.shutdown();
}
}
+
+ private static InetSocketAddress getNNSocketAddress(BPOfferService bpos) {
+ List<BPServiceActor> actors = bpos.getBPServiceActors();
+ assertEquals(1, actors.size());
+ return actors.get(0).getNNSocketAddress();
+ }
/**
* starts single nn and single dn and verifies registration and handshake
@@ -153,15 +162,16 @@ public class TestDataNodeMultipleRegistr
for (BPOfferService bpos : dn.getAllBpOs()) {
LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration.name + "; sid="
- + bpos.bpRegistration.storageID + "; nna=" + bpos.nnAddr);
+ + bpos.bpRegistration.storageID + "; nna=" +
+ getNNSocketAddress(bpos));
}
// try block report
BPOfferService bpos1 = dn.getAllBpOs()[0];
- bpos1.lastBlockReport = 0;
- bpos1.blockReport();
+ bpos1.triggerBlockReportForTests();
- assertEquals("wrong nn address", bpos1.nnAddr,
+ assertEquals("wrong nn address",
+ getNNSocketAddress(bpos1),
nn1.getNameNodeAddress());
assertEquals("wrong bpid", bpos1.getBlockPoolId(), bpid1);
assertEquals("wrong cid", dn.getClusterId(), cid1);
@@ -179,8 +189,9 @@ public class TestDataNodeMultipleRegistr
@Test
public void testClusterIdMismatch() throws IOException {
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(2).
- nameNodePort(9928).build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .build();
try {
cluster.waitActive();
@@ -215,25 +226,27 @@ public class TestDataNodeMultipleRegistr
Configuration conf = new HdfsConfiguration();
// start Federated cluster and add a node.
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(2).
- nameNodePort(9928).build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .build();
Assert.assertNotNull(cluster);
Assert.assertEquals("(1)Should be 2 namenodes", 2, cluster.getNumNameNodes());
// add a node
- cluster.addNameNode(conf, 9929);
+ cluster.addNameNode(conf, 0);
Assert.assertEquals("(1)Should be 3 namenodes", 3, cluster.getNumNameNodes());
cluster.shutdown();
// 2. start with Federation flag set
conf = new HdfsConfiguration();
- cluster = new MiniDFSCluster.Builder(conf).federation(true).
- nameNodePort(9928).build();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(1))
+ .build();
Assert.assertNotNull(cluster);
Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes());
// add a node
- cluster.addNameNode(conf, 9929);
+ cluster.addNameNode(conf, 0);
Assert.assertEquals("(2)Should be 2 namenodes", 2, cluster.getNumNameNodes());
cluster.shutdown();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java Sun Mar 11 17:55:58 2012
@@ -44,7 +44,10 @@ public class TestDatanodeRegister {
DataNode mockDN = mock(DataNode.class);
Mockito.doReturn(true).when(mockDN).shouldRun();
- BPOfferService bpos = new BPOfferService(INVALID_ADDR, mockDN);
+ BPOfferService mockBPOS = Mockito.mock(BPOfferService.class);
+ Mockito.doReturn(mockDN).when(mockBPOS).getDataNode();
+
+ BPServiceActor actor = new BPServiceActor(INVALID_ADDR, mockBPOS);
NamespaceInfo fakeNSInfo = mock(NamespaceInfo.class);
when(fakeNSInfo.getBuildVersion()).thenReturn("NSBuildVersion");
@@ -52,10 +55,9 @@ public class TestDatanodeRegister {
mock(DatanodeProtocolClientSideTranslatorPB.class);
when(fakeDNProt.versionRequest()).thenReturn(fakeNSInfo);
- bpos.setNameNode( fakeDNProt );
- bpos.bpNSInfo = fakeNSInfo;
+ actor.setNameNode( fakeDNProt );
try {
- bpos.retrieveNamespaceInfo();
+ actor.retrieveNamespaceInfo();
fail("register() did not throw exception! " +
"Expected: IncorrectVersionException");
} catch (IncorrectVersionException ie) {
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java Sun Mar 11 17:55:58 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.junit.Test;
@@ -47,8 +48,9 @@ public class TestDeleteBlockPool {
try {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES,
"namesServerId1,namesServerId2");
- cluster = new MiniDFSCluster.Builder(conf).federation(true).numNameNodes(
- 2).numDataNodes(2).build();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2).build();
cluster.waitActive();
@@ -155,8 +157,9 @@ public class TestDeleteBlockPool {
try {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES,
"namesServerId1,namesServerId2");
- cluster = new MiniDFSCluster.Builder(conf).federation(true).numNameNodes(
- 2).numDataNodes(1).build();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(1).build();
cluster.waitActive();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java Sun Mar 11 17:55:58 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.junit.Test;
@@ -41,12 +42,13 @@ public class TestMulitipleNNDataBlockSca
String bpids[] = new String[3];
FileSystem fs[] = new FileSystem[3];
- public void setUp(int port) throws IOException {
+ public void setUp() throws IOException {
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 100);
- cluster = new MiniDFSCluster.Builder(conf).numNameNodes(3).nameNodePort(
- port).build();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(3))
+ .build();
for (int i = 0; i < 3; i++) {
cluster.waitActive(i);
}
@@ -65,7 +67,7 @@ public class TestMulitipleNNDataBlockSca
@Test
public void testDataBlockScanner() throws IOException, InterruptedException {
- setUp(9923);
+ setUp();
try {
DataNode dn = cluster.getDataNodes().get(0);
for (int i = 0; i < 3; i++) {
@@ -89,9 +91,10 @@ public class TestMulitipleNNDataBlockSca
@Test
public void testBlockScannerAfterRefresh() throws IOException,
InterruptedException {
- setUp(9933);
+ setUp();
try {
- Configuration conf = new HdfsConfiguration(cluster.getConfiguration(0));
+ Configuration dnConf = cluster.getDataNodes().get(0).getConf();
+ Configuration conf = new HdfsConfiguration(dnConf);
StringBuilder namenodesBuilder = new StringBuilder();
String bpidToShutdown = cluster.getNamesystem(2).getBlockPoolId();
@@ -140,7 +143,7 @@ public class TestMulitipleNNDataBlockSca
@Test
public void testBlockScannerAfterRestart() throws IOException,
InterruptedException {
- setUp(9943);
+ setUp();
try {
cluster.restartDataNode(0);
cluster.waitActive();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java Sun Mar 11 17:55:58 2012
@@ -22,12 +22,18 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
import org.junit.Test;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+
/**
* Tests datanode refresh namenode list functionality.
*/
@@ -43,9 +49,13 @@ public class TestRefreshNamenodes {
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try {
- conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "namesServerId1");
- cluster = new MiniDFSCluster.Builder(conf).federation(true).
- numNameNodes(1).nameNodePort(nnPort1).build();
+ MiniDFSNNTopology topology = new MiniDFSNNTopology()
+ .addNameservice(new NSConf("ns1").addNN(
+ new NNConf(null).setIpcPort(nnPort1)))
+ .setFederation(true);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .build();
DataNode dn = cluster.getDataNodes().get(0);
assertEquals(1, dn.getAllBpOs().length);
@@ -58,21 +68,24 @@ public class TestRefreshNamenodes {
cluster.addNameNode(conf, nnPort4);
- BPOfferService[] bpoList = dn.getAllBpOs();
// Ensure a BPOfferService in the datanodes corresponds to
// a namenode in the cluster
+ Set<InetSocketAddress> nnAddrsFromCluster = Sets.newHashSet();
for (int i = 0; i < 4; i++) {
- InetSocketAddress addr = cluster.getNameNode(i).getNameNodeAddress();
- boolean found = false;
- for (int j = 0; j < bpoList.length; j++) {
- if (bpoList[j] != null && addr.equals(bpoList[j].nnAddr)) {
- found = true;
- bpoList[j] = null; // Erase the address that matched
- break;
- }
+ assertTrue(nnAddrsFromCluster.add(
+ cluster.getNameNode(i).getNameNodeAddress()));
+ }
+
+ Set<InetSocketAddress> nnAddrsFromDN = Sets.newHashSet();
+ for (BPOfferService bpos : dn.getAllBpOs()) {
+ for (BPServiceActor bpsa : bpos.getBPServiceActors()) {
+ assertTrue(nnAddrsFromDN.add(bpsa.getNNSocketAddress()));
}
- assertTrue("NameNode address " + addr + " is not found.", found);
}
+
+ assertEquals("",
+ Joiner.on(",").join(
+ Sets.symmetricDifference(nnAddrsFromCluster, nnAddrsFromDN)));
} finally {
if (cluster != null) {
cluster.shutdown();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java Sun Mar 11 17:55:58 2012
@@ -195,7 +195,7 @@ public class CreateEditsLog {
FileNameGenerator nameGenerator = new FileNameGenerator(BASE_PATH, 100);
FSEditLog editLog = FSImageTestUtil.createStandaloneEditLog(editsLogDir);
- editLog.open();
+ editLog.openForWrite();
addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId,
nameGenerator);
editLog.logSync();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Sun Mar 11 17:55:58 2012
@@ -34,8 +34,11 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -187,13 +190,36 @@ public abstract class FSImageTestUtil {
Mockito.doReturn(sd).when(storage)
.getStorageDirectory(Matchers.<URI>anyObject());
- return new FSEditLog(new Configuration(),
+ FSEditLog editLog = new FSEditLog(new Configuration(),
storage,
ImmutableList.of(logDir.toURI()));
+ editLog.initJournalsForWrite();
+ return editLog;
}
/**
+ * Create an aborted in-progress log in the given directory, containing
+ * only a specified number of "mkdirs" operations.
+ */
+ public static void createAbortedLogWithMkdirs(File editsLogDir, int numDirs,
+ long firstTxId) throws IOException {
+ FSEditLog editLog = FSImageTestUtil.createStandaloneEditLog(editsLogDir);
+ editLog.setNextTxId(firstTxId);
+ editLog.openForWrite();
+
+ PermissionStatus perms = PermissionStatus.createImmutable("fakeuser", "fakegroup",
+ FsPermission.createImmutable((short)0755));
+ for (int i = 1; i <= numDirs; i++) {
+ String dirName = "dir" + i;
+ INodeDirectory dir = new INodeDirectory(dirName, perms);
+ editLog.logMkDir("/" + dirName, dir);
+ }
+ editLog.logSync();
+ editLog.abortCurrentLogSegment();
+ }
+
+ /**
* @param editLog a path of an edit log file
* @return the count of each type of operation in the log file
* @throws Exception if there is an error reading it
@@ -410,13 +436,20 @@ public abstract class FSImageTestUtil {
* Assert that the NameNode has checkpoints at the expected
* transaction IDs.
*/
- static void assertNNHasCheckpoints(MiniDFSCluster cluster,
+ public static void assertNNHasCheckpoints(MiniDFSCluster cluster,
List<Integer> txids) {
+ assertNNHasCheckpoints(cluster, 0, txids);
+ }
+
+ public static void assertNNHasCheckpoints(MiniDFSCluster cluster,
+ int nnIdx, List<Integer> txids) {
- for (File nameDir : getNameNodeCurrentDirs(cluster)) {
+ for (File nameDir : getNameNodeCurrentDirs(cluster, nnIdx)) {
LOG.info("examining name dir with files: " +
Joiner.on(",").join(nameDir.listFiles()));
// Should have fsimage_N for the three checkpoints
+ LOG.info("Examining storage dir " + nameDir + " with contents: "
+ + StringUtils.join(nameDir.listFiles(), ", "));
for (long checkpointTxId : txids) {
File image = new File(nameDir,
NNStorage.getImageFileName(checkpointTxId));
@@ -425,9 +458,9 @@ public abstract class FSImageTestUtil {
}
}
- public static List<File> getNameNodeCurrentDirs(MiniDFSCluster cluster) {
+ public static List<File> getNameNodeCurrentDirs(MiniDFSCluster cluster, int nnIdx) {
List<File> nameDirs = Lists.newArrayList();
- for (URI u : cluster.getNameDirs(0)) {
+ for (URI u : cluster.getNameDirs(nnIdx)) {
nameDirs.add(new File(u.getPath(), "current"));
}
return nameDirs;
@@ -441,7 +474,7 @@ public abstract class FSImageTestUtil {
throws IOException {
File currentDir = sd.getCurrentDir();
List<EditLogFile> foundEditLogs
- = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir.listFiles()));
+ = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir));
return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1299412&r1=1299411&r2=1299412&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Sun Mar 11 17:55:58 2012
@@ -80,7 +80,7 @@ import org.apache.log4j.LogManager;
* <li>-logLevel L specifies the logging level when the benchmark runs.
* The default logging level is {@link Level#ERROR}.</li>
* <li>-UGCacheRefreshCount G will cause the benchmark to call
- * {@link NameNode#refreshUserToGroupsMappings()} after
+ * {@link NameNodeRpcServer#refreshUserToGroupsMappings} after
* every G operations, which purges the name-node's user group cache.
* By default the refresh is never called.</li>
* <li>-keepResults do not clean up the name-space after execution.</li>
@@ -813,7 +813,7 @@ public class NNThroughputBenchmark {
StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
- rep, 0, 0, 0);
+ rep, 0, 0, 0).getCommands();
if(cmds != null) {
for (DatanodeCommand cmd : cmds ) {
if(LOG.isDebugEnabled()) {
@@ -859,7 +859,7 @@ public class NNThroughputBenchmark {
StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
- rep, 0, 0, 0);
+ rep, 0, 0, 0).getCommands();
if (cmds != null) {
for (DatanodeCommand cmd : cmds) {
if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
@@ -889,8 +889,10 @@ public class NNThroughputBenchmark {
receivedDNReg.setStorageInfo(
new DataStorage(nsInfo, dnInfo.getStorageID()));
receivedDNReg.setInfoPort(dnInfo.getInfoPort());
- ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo(
- blocks[i], DataNode.EMPTY_DEL_HINT) };
+ ReceivedDeletedBlockInfo[] rdBlocks = {
+ new ReceivedDeletedBlockInfo(
+ blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
+ null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
receivedDNReg.getStorageID(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode
@@ -1007,7 +1009,8 @@ public class NNThroughputBenchmark {
int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getName());
datanodes[dnIdx].addBlock(loc.getBlock().getLocalBlock());
ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo(
- loc.getBlock().getLocalBlock(), "") };
+ loc.getBlock().getLocalBlock(),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc