You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/03/03 01:43:00 UTC
svn commit: r1296534 [10/11] - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ dev-support/
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/
sr...
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Sat Mar 3 00:42:49 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -123,6 +124,7 @@ public class TestBackupNode {
@Test
public void testBackupNodeTailsEdits() throws Exception {
Configuration conf = new HdfsConfiguration();
+ HAUtil.setAllowStandbyReads(conf, true);
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
BackupNode backup = null;
@@ -244,11 +246,12 @@ public class TestBackupNode {
}
void testCheckpoint(StartupOption op) throws Exception {
- Path file1 = new Path("checkpoint.dat");
- Path file2 = new Path("checkpoint2.dat");
- Path file3 = new Path("backup.dat");
+ Path file1 = new Path("/checkpoint.dat");
+ Path file2 = new Path("/checkpoint2.dat");
+ Path file3 = new Path("/backup.dat");
Configuration conf = new HdfsConfiguration();
+ HAUtil.setAllowStandbyReads(conf, true);
short replication = (short)conf.getInt("dfs.replication", 3);
int numDatanodes = Math.max(3, replication);
conf.set(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY, "0");
@@ -345,11 +348,13 @@ public class TestBackupNode {
TestCheckpoint.checkFile(fileSys, file3, replication);
// should also be on BN right away
assertTrue("file3 does not exist on BackupNode",
- op != StartupOption.BACKUP || bnFS.exists(file3));
+ op != StartupOption.BACKUP ||
+ backup.getNamesystem().getFileInfo(
+ file3.toUri().getPath(), false) != null);
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
- assertTrue(e.getLocalizedMessage(), false);
+ throw new AssertionError(e);
} finally {
if(backup != null) backup.stop();
if(fileSys != null) fileSys.close();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java Sat Mar 3 00:42:49 2012
@@ -22,6 +22,7 @@ import junit.framework.Assert;
import java.io.*;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -57,18 +58,19 @@ public class TestCheckPointForSecurityTo
}
/**
- * Tests save namepsace.
+ * Tests save namespace.
*/
@Test
public void testSaveNamespace() throws IOException {
DistributedFileSystem fs = null;
try {
Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
cluster.waitActive();
fs = (DistributedFileSystem)(cluster.getFileSystem());
FSNamesystem namesystem = cluster.getNamesystem();
- namesystem.getDelegationTokenSecretManager().startThreads();
String renewer = UserGroupInformation.getLoginUser().getUserName();
Token<DelegationTokenIdentifier> token1 = namesystem
.getDelegationToken(new Text(renewer));
@@ -122,7 +124,6 @@ public class TestCheckPointForSecurityTo
}
namesystem = cluster.getNamesystem();
- namesystem.getDelegationTokenSecretManager().startThreads();
Token<DelegationTokenIdentifier> token3 = namesystem
.getDelegationToken(new Text(renewer));
Token<DelegationTokenIdentifier> token4 = namesystem
@@ -136,7 +137,6 @@ public class TestCheckPointForSecurityTo
cluster.waitActive();
namesystem = cluster.getNamesystem();
- namesystem.getDelegationTokenSecretManager().startThreads();
Token<DelegationTokenIdentifier> token5 = namesystem
.getDelegationToken(new Text(renewer));
@@ -159,7 +159,6 @@ public class TestCheckPointForSecurityTo
cluster.waitActive();
namesystem = cluster.getNamesystem();
- namesystem.getDelegationTokenSecretManager().startThreads();
try {
renewToken(token1);
cancelToken(token1);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Sat Mar 3 00:42:49 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSUtil.Er
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -655,6 +656,7 @@ public class TestCheckpoint extends Test
sdToLock.lock();
try {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .format(false)
.manageNameDfsDirs(false)
.numDataNodes(0)
.build();
@@ -861,7 +863,7 @@ public class TestCheckpoint extends Test
}
/**
- * Tests save namepsace.
+ * Tests save namespace.
*/
public void testSaveNamespace() throws IOException {
MiniDFSCluster cluster = null;
@@ -911,10 +913,12 @@ public class TestCheckpoint extends Test
throw new IOException(e);
}
+ final int EXPECTED_TXNS_FIRST_SEG = 12;
+
// the following steps should have happened:
- // edits_inprogress_1 -> edits_1-8 (finalized)
- // fsimage_8 created
- // edits_inprogress_9 created
+ // edits_inprogress_1 -> edits_1-12 (finalized)
+ // fsimage_12 created
+ // edits_inprogress_13 created
//
for(URI uri : editsDirs) {
File ed = new File(uri.getPath());
@@ -926,19 +930,21 @@ public class TestCheckpoint extends Test
NNStorage.getInProgressEditsFileName(1));
assertFalse(originalEdits.exists());
File finalizedEdits = new File(curDir,
- NNStorage.getFinalizedEditsFileName(1,8));
- assertTrue(finalizedEdits.exists());
+ NNStorage.getFinalizedEditsFileName(1, EXPECTED_TXNS_FIRST_SEG));
+ GenericTestUtils.assertExists(finalizedEdits);
assertTrue(finalizedEdits.length() > Integer.SIZE/Byte.SIZE);
- assertTrue(new File(ed, "current/"
- + NNStorage.getInProgressEditsFileName(9)).exists());
+ GenericTestUtils.assertExists(new File(ed, "current/"
+ + NNStorage.getInProgressEditsFileName(
+ EXPECTED_TXNS_FIRST_SEG + 1)));
}
Collection<URI> imageDirs = cluster.getNameDirs(0);
for (URI uri : imageDirs) {
File imageDir = new File(uri.getPath());
File savedImage = new File(imageDir, "current/"
- + NNStorage.getImageFileName(8));
+ + NNStorage.getImageFileName(
+ EXPECTED_TXNS_FIRST_SEG));
assertTrue("Should have saved image at " + savedImage,
savedImage.exists());
}
@@ -1059,8 +1065,9 @@ public class TestCheckpoint extends Test
String nameserviceId2 = "ns2";
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, nameserviceId1
+ "," + nameserviceId2);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(2)
- .nameNodePort(9928).build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .build();
Configuration snConf1 = new HdfsConfiguration(cluster.getConfiguration(0));
Configuration snConf2 = new HdfsConfiguration(cluster.getConfiguration(1));
InetSocketAddress nn1RpcAddress =
@@ -1076,9 +1083,9 @@ public class TestCheckpoint extends Test
snConf2.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "");
// Set the nameserviceIds
- snConf1.set(DFSUtil.getNameServiceIdKey(
+ snConf1.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nameserviceId1), nn1);
- snConf2.set(DFSUtil.getNameServiceIdKey(
+ snConf2.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nameserviceId2), nn2);
SecondaryNameNode secondary1 = startSecondaryNameNode(snConf1);
@@ -1317,17 +1324,11 @@ public class TestCheckpoint extends Test
// Let the first one finish
delayer.proceed();
- // Letting the first node continue should catch an exception
+ // Letting the first node continue, it should try to upload the
+ // same image, and gracefully ignore it, while logging an
+ // error message.
checkpointThread.join();
- try {
- checkpointThread.propagateExceptions();
- fail("Didn't throw!");
- } catch (Exception ioe) {
- assertTrue("Unexpected exception: " +
- StringUtils.stringifyException(ioe),
- ioe.toString().contains("Another checkpointer already uploaded"));
- LOG.info("Caught expected exception", ioe);
- }
+ checkpointThread.propagateExceptions();
// primary should still consider fsimage_4 the latest
assertEquals(4, storage.getMostRecentCheckpointTxId());
@@ -1763,7 +1764,7 @@ public class TestCheckpoint extends Test
private void assertParallelFilesInvariant(MiniDFSCluster cluster,
ImmutableList<SecondaryNameNode> secondaries) throws Exception {
List<File> allCurrentDirs = Lists.newArrayList();
- allCurrentDirs.addAll(getNameNodeCurrentDirs(cluster));
+ allCurrentDirs.addAll(getNameNodeCurrentDirs(cluster, 0));
for (SecondaryNameNode snn : secondaries) {
allCurrentDirs.addAll(getCheckpointCurrentDirs(snn));
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java Sat Mar 3 00:42:49 2012
@@ -26,6 +26,7 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Properties;
import org.apache.commons.logging.Log;
@@ -47,7 +48,7 @@ public class TestClusterId {
private String getClusterId(Configuration config) throws IOException {
// see if cluster id not empty.
Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(config);
- Collection<URI> editsToFormat = FSNamesystem.getNamespaceEditsDirs(config);
+ List<URI> editsToFormat = FSNamesystem.getNamespaceEditsDirs(config);
FSImage fsImage = new FSImage(config, dirsToFormat, editsToFormat);
Iterator<StorageDirectory> sdit =
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java Sat Mar 3 00:42:49 2012
@@ -110,7 +110,9 @@ public class TestDeadDatanode {
DatanodeProtocol dnp = cluster.getNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = { new ReceivedDeletedBlockInfo(
- new Block(0), "") };
+ new Block(0),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
+ null) };
StorageReceivedDeletedBlocks[] storageBlocks = {
new StorageReceivedDeletedBlocks(reg.getStorageID(), blocks) };
@@ -136,7 +138,7 @@ public class TestDeadDatanode {
// that asks datanode to register again
StorageReport[] rep = { new StorageReport(reg.getStorageID(), false, 0, 0,
0, 0) };
- DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0, 0, 0);
+ DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0, 0, 0).getCommands();
Assert.assertEquals(1, cmd.length);
Assert.assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
.getAction());
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Sat Mar 3 00:42:49 2012
@@ -147,7 +147,7 @@ public class TestEditLog extends TestCas
public void testPreTxIdEditLogNoEdits() throws Exception {
FSNamesystem namesys = Mockito.mock(FSNamesystem.class);
namesys.dir = Mockito.mock(FSDirectory.class);
- int numEdits = testLoad(
+ long numEdits = testLoad(
StringUtils.hexStringToByte("ffffffed"), // just version number
namesys);
assertEquals(0, numEdits);
@@ -166,7 +166,7 @@ public class TestEditLog extends TestCas
cluster.waitActive();
final FSNamesystem namesystem = cluster.getNamesystem();
- int numEdits = testLoad(HADOOP20_SOME_EDITS, namesystem);
+ long numEdits = testLoad(HADOOP20_SOME_EDITS, namesystem);
assertEquals(3, numEdits);
// Sanity check the edit
HdfsFileStatus fileInfo = namesystem.getFileInfo("/myfile", false);
@@ -177,7 +177,7 @@ public class TestEditLog extends TestCas
}
}
- private int testLoad(byte[] data, FSNamesystem namesys) throws IOException {
+ private long testLoad(byte[] data, FSNamesystem namesys) throws IOException {
FSEditLogLoader loader = new FSEditLogLoader(namesys);
return loader.loadFSEdits(new EditLogByteInputStream(data), 1);
}
@@ -321,7 +321,7 @@ public class TestEditLog extends TestCas
assertTrue("Expect " + editFile + " exists", editFile.exists());
System.out.println("Verifying file: " + editFile);
- int numEdits = loader.loadFSEdits(
+ long numEdits = loader.loadFSEdits(
new EditLogFileInputStream(editFile), 3);
int numLeases = namesystem.leaseManager.countLease();
System.out.println("Number of outstanding leases " + numLeases);
@@ -589,7 +589,6 @@ public class TestEditLog extends TestCas
currentDir.getAbsolutePath());
assertNotNull("No image found in " + nameDir, imageFile);
assertEquals(NNStorage.getImageFileName(0), imageFile.getName());
-
// Try to start a new cluster
LOG.info("\n===========================================\n" +
"Starting same cluster after simulated crash");
@@ -636,22 +635,26 @@ public class TestEditLog extends TestCas
}
}
+ // should succeed - only one corrupt log dir
public void testCrashRecoveryEmptyLogOneDir() throws Exception {
- doTestCrashRecoveryEmptyLog(false, true);
+ doTestCrashRecoveryEmptyLog(false, true, true);
}
+ // should fail - seen_txid updated to 3, but no log dir contains txid 3
public void testCrashRecoveryEmptyLogBothDirs() throws Exception {
- doTestCrashRecoveryEmptyLog(true, true);
+ doTestCrashRecoveryEmptyLog(true, true, false);
}
+ // should succeed - only one corrupt log dir
public void testCrashRecoveryEmptyLogOneDirNoUpdateSeenTxId()
throws Exception {
- doTestCrashRecoveryEmptyLog(false, false);
+ doTestCrashRecoveryEmptyLog(false, false, true);
}
+ // should succeed - both log dirs corrupt, but seen_txid never updated
public void testCrashRecoveryEmptyLogBothDirsNoUpdateSeenTxId()
throws Exception {
- doTestCrashRecoveryEmptyLog(true, false);
+ doTestCrashRecoveryEmptyLog(true, false, true);
}
/**
@@ -667,12 +670,13 @@ public class TestEditLog extends TestCas
* NN should fail to start up, because it's aware that txid 3
* was reached, but unable to find a non-corrupt log starting there.
* @param updateTransactionIdFile if true update the seen_txid file.
- * If false, the it will not be updated. This will simulate a case
- * where the NN crashed between creating the new segment and updating
- * seen_txid.
+ * If false, it will not be updated. This will simulate a case where
+ * the NN crashed between creating the new segment and updating the
+ * seen_txid file.
+ * @param shouldSucceed true if the test is expected to succeed.
*/
private void doTestCrashRecoveryEmptyLog(boolean inBothDirs,
- boolean updateTransactionIdFile)
+ boolean updateTransactionIdFile, boolean shouldSucceed)
throws Exception {
// start a cluster
Configuration conf = new HdfsConfiguration();
@@ -691,29 +695,40 @@ public class TestEditLog extends TestCas
// Make a truncated edits_3_inprogress
File log = new File(currentDir,
NNStorage.getInProgressEditsFileName(3));
- NNStorage storage = new NNStorage(conf,
- Collections.<URI>emptyList(),
- Lists.newArrayList(uri));
- if (updateTransactionIdFile) {
- storage.writeTransactionIdFileToStorage(3);
- }
- storage.close();
new EditLogFileOutputStream(log, 1024).create();
if (!inBothDirs) {
break;
}
+
+ NNStorage storage = new NNStorage(conf,
+ Collections.<URI>emptyList(),
+ Lists.newArrayList(uri));
+
+ if (updateTransactionIdFile) {
+ storage.writeTransactionIdFileToStorage(3);
+ }
+ storage.close();
}
try {
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATA_NODES).format(false).build();
- fail("Did not fail to start with all-corrupt logs");
+ if (!shouldSucceed) {
+ fail("Should not have succeeded in startin cluster");
+ }
} catch (IOException ioe) {
- GenericTestUtils.assertExceptionContains(
- "No non-corrupt logs for txid 3", ioe);
+ if (shouldSucceed) {
+ LOG.info("Should have succeeded in starting cluster, but failed", ioe);
+ throw ioe;
+ } else {
+ GenericTestUtils.assertExceptionContains(
+ "No non-corrupt logs for txid 3",
+ ioe);
+ }
+ } finally {
+ cluster.shutdown();
}
- cluster.shutdown();
}
@@ -781,6 +796,11 @@ public class TestEditLog extends TestCas
public JournalType getType() {
return JournalType.FILE;
}
+
+ @Override
+ public boolean isInProgress() {
+ return true;
+ }
}
public void testFailedOpen() throws Exception {
@@ -789,11 +809,11 @@ public class TestEditLog extends TestCas
FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
try {
logDir.setWritable(false);
- log.open();
+ log.openForWrite();
fail("Did no throw exception on only having a bad dir");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
- "no journals successfully started", ioe);
+ "too few journals successfully started", ioe);
} finally {
logDir.setWritable(true);
log.close();
@@ -813,7 +833,7 @@ public class TestEditLog extends TestCas
new byte[500]);
try {
- log.open();
+ log.openForWrite();
NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
log.setMetricsForTests(mockMetrics);
@@ -848,6 +868,7 @@ public class TestEditLog extends TestCas
"[1,100]|[101,200]|[201,]",
"[1,100]|[101,200]|[201,]");
log = new FSEditLog(storage);
+ log.initJournalsForWrite();
assertEquals("[[1,100], [101,200]]",
log.getEditLogManifest(1).toString());
assertEquals("[[101,200]]",
@@ -859,6 +880,7 @@ public class TestEditLog extends TestCas
"[1,100]|[101,200]",
"[1,100]|[201,300]|[301,400]"); // nothing starting at 101
log = new FSEditLog(storage);
+ log.initJournalsForWrite();
assertEquals("[[1,100], [101,200], [201,300], [301,400]]",
log.getEditLogManifest(1).toString());
@@ -868,6 +890,7 @@ public class TestEditLog extends TestCas
"[1,100]|[301,400]", // gap from 101 to 300
"[301,400]|[401,500]");
log = new FSEditLog(storage);
+ log.initJournalsForWrite();
assertEquals("[[301,400], [401,500]]",
log.getEditLogManifest(1).toString());
@@ -877,6 +900,7 @@ public class TestEditLog extends TestCas
"[1,100]|[101,150]", // short log at 101
"[1,50]|[101,200]"); // short log at 1
log = new FSEditLog(storage);
+ log.initJournalsForWrite();
assertEquals("[[1,100], [101,200]]",
log.getEditLogManifest(1).toString());
assertEquals("[[101,200]]",
@@ -889,6 +913,7 @@ public class TestEditLog extends TestCas
"[1,100]|[101,]",
"[1,100]|[101,200]");
log = new FSEditLog(storage);
+ log.initJournalsForWrite();
assertEquals("[[1,100], [101,200]]",
log.getEditLogManifest(1).toString());
assertEquals("[[101,200]]",
@@ -967,11 +992,11 @@ public class TestEditLog extends TestCas
*
* @param editUris directories to create edit logs in
* @param numrolls number of times to roll the edit log during setup
+ * @param closeOnFinish whether to close the edit log after setup
* @param abortAtRolls Specifications for when to fail, see AbortSpec
*/
- public static NNStorage setupEdits(List<URI> editUris, int numrolls,
- AbortSpec... abortAtRolls)
- throws IOException {
+ public static NNStorage setupEdits(List<URI> editUris, int numrolls,
+ boolean closeOnFinish, AbortSpec... abortAtRolls) throws IOException {
List<AbortSpec> aborts = new ArrayList<AbortSpec>(Arrays.asList(abortAtRolls));
NNStorage storage = new NNStorage(new Configuration(),
Collections.<URI>emptyList(),
@@ -981,7 +1006,8 @@ public class TestEditLog extends TestCas
// open the edit log and add two transactions
// logGenerationStamp is used, simply because it doesn't
// require complex arguments.
- editlog.open();
+ editlog.initJournalsForWrite();
+ editlog.openForWrite();
for (int i = 2; i < TXNS_PER_ROLL; i++) {
editlog.logGenerationStamp((long)0);
}
@@ -1009,16 +1035,34 @@ public class TestEditLog extends TestCas
}
editlog.logSync();
}
- editlog.close();
+
+ if (closeOnFinish) {
+ editlog.close();
+ }
FSImageTestUtil.logStorageContents(LOG, storage);
return storage;
}
+
+ /**
+ * Set up directories for tests.
+ *
+ * Each rolled file is 10 txns long.
+ * A failed file is 2 txns long.
+ *
+ * @param editUris directories to create edit logs in
+ * @param numrolls number of times to roll the edit log during setup
+ * @param abortAtRolls Specifications for when to fail, see AbortSpec
+ */
+ public static NNStorage setupEdits(List<URI> editUris, int numrolls,
+ AbortSpec... abortAtRolls) throws IOException {
+ return setupEdits(editUris, numrolls, true, abortAtRolls);
+ }
/**
* Test loading an editlog which has had both its storage fail
* on alternating rolls. Two edit log directories are created.
- * The first on fails on odd rolls, the second on even. Test
+ * The first one fails on odd rolls, the second on even. Test
* that we are able to load the entire editlog regardless.
*/
@Test
@@ -1041,6 +1085,7 @@ public class TestEditLog extends TestCas
new AbortSpec(10, 1));
long totaltxnread = 0;
FSEditLog editlog = new FSEditLog(storage);
+ editlog.initJournalsForWrite();
long startTxId = 1;
Iterable<EditLogInputStream> editStreams = editlog.selectInputStreams(startTxId,
TXNS_PER_ROLL*11);
@@ -1090,11 +1135,10 @@ public class TestEditLog extends TestCas
assertTrue(files[0].delete());
FSEditLog editlog = new FSEditLog(storage);
+ editlog.initJournalsForWrite();
long startTxId = 1;
try {
- Iterable<EditLogInputStream> editStreams
- = editlog.selectInputStreams(startTxId, 4*TXNS_PER_ROLL);
-
+ editlog.selectInputStreams(startTxId, 4*TXNS_PER_ROLL);
fail("Should have thrown exception");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java Sat Mar 3 00:42:49 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.verification.VerificationMode;
public class TestEditLogJournalFailures {
@@ -144,21 +145,35 @@ public class TestEditLogJournalFailures
DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
shutDownMiniCluster();
Configuration conf = new HdfsConfiguration();
- conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, editsDirs[1]);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, editsDirs[0]);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY, 0);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY, 0);
setUpMiniCluster(conf, true);
assertTrue(doAnEdit());
// Invalidated the one required edits journal.
- invalidateEditsDirAtIndex(1, false, false);
+ invalidateEditsDirAtIndex(0, false, false);
+ JournalAndStream nonRequiredJas = getJournalAndStream(1);
+ EditLogFileOutputStream nonRequiredSpy =
+ spyOnStream(nonRequiredJas);
+
// Make sure runtime.exit(...) hasn't been called at all yet.
assertExitInvocations(0);
+ // ..and that the other stream is active.
+ assertTrue(nonRequiredJas.isActive());
+
// This will actually return true in the tests, since the NN will not in
// fact call Runtime.exit();
doAnEdit();
+ // Since the required directory failed setReadyToFlush, and that
+ // directory was listed prior to the non-required directory,
+ // we should not call setReadyToFlush on the non-required
+ // directory. Regression test for HDFS-2874.
+ Mockito.verify(nonRequiredSpy, Mockito.never()).setReadyToFlush();
+ assertFalse(nonRequiredJas.isActive());
+
// A single failure of a required journal should result in a call to
// runtime.exit(...).
assertExitInvocations(atLeast(1));
@@ -217,15 +232,10 @@ public class TestEditLogJournalFailures
* @param index the index of the journal to take offline.
* @return the original <code>EditLogOutputStream</code> of the journal.
*/
- private EditLogOutputStream invalidateEditsDirAtIndex(int index,
+ private void invalidateEditsDirAtIndex(int index,
boolean failOnFlush, boolean failOnWrite) throws IOException {
- FSImage fsimage = cluster.getNamesystem().getFSImage();
- FSEditLog editLog = fsimage.getEditLog();
-
- JournalAndStream jas = editLog.getJournals().get(index);
- EditLogFileOutputStream elos =
- (EditLogFileOutputStream) jas.getCurrentStream();
- EditLogFileOutputStream spyElos = spy(elos);
+ JournalAndStream jas = getJournalAndStream(index);
+ EditLogFileOutputStream spyElos = spyOnStream(jas);
if (failOnWrite) {
doThrow(new IOException("fail on write()")).when(spyElos).write(
(FSEditLogOp) any());
@@ -237,25 +247,24 @@ public class TestEditLogJournalFailures
.setReadyToFlush();
}
doNothing().when(spyElos).abort();
-
+ }
+
+ private EditLogFileOutputStream spyOnStream(JournalAndStream jas) {
+ EditLogFileOutputStream elos =
+ (EditLogFileOutputStream) jas.getCurrentStream();
+ EditLogFileOutputStream spyElos = spy(elos);
jas.setCurrentStreamForTests(spyElos);
-
- return elos;
+ return spyElos;
}
/**
- * Restore the journal at index <code>index</code> with the passed
- * {@link EditLogOutputStream}.
- *
- * @param index index of the journal to restore.
- * @param elos the {@link EditLogOutputStream} to put at that index.
+ * Pull out one of the JournalAndStream objects from the edit log.
*/
- private void restoreEditsDirAtIndex(int index, EditLogOutputStream elos) {
+ private JournalAndStream getJournalAndStream(int index) {
FSImage fsimage = cluster.getNamesystem().getFSImage();
FSEditLog editLog = fsimage.getEditLog();
- JournalAndStream jas = editLog.getJournals().get(index);
- jas.setCurrentStreamForTests(elos);
+ return editLog.getJournals().get(index);
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java Sat Mar 3 00:42:49 2012
@@ -237,7 +237,7 @@ public class TestEditLogRace {
System.out.println("Verifying file: " + editFile);
FSEditLogLoader loader = new FSEditLogLoader(namesystem);
- int numEditsThisLog = loader.loadFSEdits(new EditLogFileInputStream(editFile),
+ long numEditsThisLog = loader.loadFSEdits(new EditLogFileInputStream(editFile),
startTxId);
System.out.println("Number of edits: " + numEditsThisLog);
@@ -375,6 +375,7 @@ public class TestEditLogRace {
true);
LOG.info("mkdirs complete");
} catch (Throwable ioe) {
+ LOG.fatal("Got exception", ioe);
deferredException.set(ioe);
waitToEnterFlush.countDown();
}
@@ -469,6 +470,7 @@ public class TestEditLogRace {
true);
LOG.info("mkdirs complete");
} catch (Throwable ioe) {
+ LOG.fatal("Got exception", ioe);
deferredException.set(ioe);
waitToEnterSync.countDown();
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java Sat Mar 3 00:42:49 2012
@@ -92,7 +92,7 @@ public class TestFSEditLogLoader {
StringBuilder bld = new StringBuilder();
bld.append("^Error replaying edit log at offset \\d+");
- bld.append("On transaction ID \\d+\n");
+ bld.append(" on transaction ID \\d+\n");
bld.append("Recent opcode offsets: (\\d+\\s*){4}$");
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
@@ -167,7 +167,7 @@ public class TestFSEditLogLoader {
SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
try {
fsel = FSImageTestUtil.createStandaloneEditLog(testDir);
- fsel.open();
+ fsel.openForWrite();
assertTrue("should exist: " + logFile, logFile.exists());
for (int i = 0; i < NUM_TXNS; i++) {
@@ -245,7 +245,9 @@ public class TestFSEditLogLoader {
Files.copy(logFileBak, logFile);
corruptByteInFile(logFile, offset);
EditLogValidation val = EditLogFileInputStream.validateEditLog(logFile);
- assertTrue(val.getNumTransactions() >= prevNumValid);
+ assertTrue(String.format("%d should have been >= %d",
+ val.getNumTransactions(), prevNumValid),
+ val.getNumTransactions() >= prevNumValid);
prevNumValid = val.getNumTransactions();
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Sat Mar 3 00:42:49 2012
@@ -29,7 +29,9 @@ import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import org.junit.Test;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.JournalManager.CorruptionException;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.setupEdits;
@@ -58,8 +60,8 @@ public class TestFileJournalManager {
long numJournals = 0;
for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
- FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
numJournals++;
}
assertEquals(3, numJournals);
@@ -78,9 +80,9 @@ public class TestFileJournalManager {
5, new AbortSpec(5, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
- FileJournalManager jm = new FileJournalManager(sd);
+ FileJournalManager jm = new FileJournalManager(sd, storage);
assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
- jm.getNumberOfTransactions(1));
+ jm.getNumberOfTransactions(1, true));
}
/**
@@ -101,16 +103,17 @@ public class TestFileJournalManager {
5, new AbortSpec(5, 1));
Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
StorageDirectory sd = dirs.next();
- FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
sd = dirs.next();
- jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
+ jm = new FileJournalManager(sd, storage);
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
- jm = new FileJournalManager(sd);
- assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
+ jm = new FileJournalManager(sd, storage);
+ assertEquals(6*TXNS_PER_ROLL, jm.getNumberOfTransactions(1, true));
}
/**
@@ -133,16 +136,19 @@ public class TestFileJournalManager {
new AbortSpec(5, 2));
Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
StorageDirectory sd = dirs.next();
- FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
- jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
+ jm = new FileJournalManager(sd, storage);
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
sd = dirs.next();
- jm = new FileJournalManager(sd);
- assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
+ jm = new FileJournalManager(sd, storage);
+ assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1,
+ true));
}
/**
@@ -156,6 +162,25 @@ public class TestFileJournalManager {
}
raf.close();
}
+
+ @Test(expected=IllegalStateException.class)
+ public void testFinalizeErrorReportedToNNStorage() throws IOException, InterruptedException {
+ File f = new File(TestEditLog.TEST_DIR + "/filejournaltestError");
+ // abort after 10th roll
+ NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
+ 10, new AbortSpec(10, 0));
+ StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
+
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+ String sdRootPath = sd.getRoot().getAbsolutePath();
+ FileUtil.chmod(sdRootPath, "-w", true);
+ try {
+ jm.finalizeLogSegment(0, 1);
+ } finally {
+ assertTrue(storage.getRemovedStorageDirs().contains(sd));
+ FileUtil.chmod(sdRootPath, "+w", true);
+ }
+ }
/**
* Test that we can read from a stream created by FileJournalManager.
@@ -171,17 +196,17 @@ public class TestFileJournalManager {
10, new AbortSpec(10, 0));
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
- FileJournalManager jm = new FileJournalManager(sd);
+ FileJournalManager jm = new FileJournalManager(sd, storage);
long expectedTotalTxnCount = TXNS_PER_ROLL*10 + TXNS_PER_FAIL;
- assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1));
+ assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1, true));
long skippedTxns = (3*TXNS_PER_ROLL); // skip first 3 files
long startingTxId = skippedTxns + 1;
- long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId);
+ long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId, true);
long numLoaded = 0;
while (numLoaded < numTransactionsToLoad) {
- EditLogInputStream editIn = jm.getInputStream(startingTxId);
+ EditLogInputStream editIn = jm.getInputStream(startingTxId, true);
FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(editIn);
long count = val.getNumTransactions();
@@ -194,20 +219,26 @@ public class TestFileJournalManager {
}
/**
- * Try to make a request with a start transaction id which doesn't
- * match the start ID of some log segment.
- * This should fail as edit logs must currently be treated as indevisable
- * units.
+ * Make requests with starting transaction ids which don't match the beginning
+ * txid of some log segments.
+ *
+ * This should succeed.
*/
- @Test(expected=IOException.class)
+ @Test
public void testAskForTransactionsMidfile() throws IOException {
File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
10);
StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
- FileJournalManager jm = new FileJournalManager(sd);
- jm.getNumberOfTransactions(2);
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+
+ // 10 rolls, so 11 rolled files, 110 txids total.
+ final int TOTAL_TXIDS = 10 * 11;
+ for (int txid = 1; txid <= TOTAL_TXIDS; txid++) {
+ assertEquals((TOTAL_TXIDS - txid) + 1, jm.getNumberOfTransactions(txid,
+ true));
+ }
}
/**
@@ -237,19 +268,20 @@ public class TestFileJournalManager {
assertEquals(1, files.length);
assertTrue(files[0].delete());
- FileJournalManager jm = new FileJournalManager(sd);
- assertEquals(startGapTxId-1, jm.getNumberOfTransactions(1));
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+ assertEquals(startGapTxId-1, jm.getNumberOfTransactions(1, true));
try {
- jm.getNumberOfTransactions(startGapTxId);
+ jm.getNumberOfTransactions(startGapTxId, true);
fail("Should have thrown an exception by now");
} catch (IOException ioe) {
- assertTrue(true);
+ GenericTestUtils.assertExceptionContains(
+ "Gap in transactions, max txnid is 110, 0 txns from 31", ioe);
}
// rolled 10 times so there should be 11 files.
assertEquals(11*TXNS_PER_ROLL - endGapTxId,
- jm.getNumberOfTransactions(endGapTxId+1));
+ jm.getNumberOfTransactions(endGapTxId + 1, true));
}
/**
@@ -274,9 +306,9 @@ public class TestFileJournalManager {
corruptAfterStartSegment(files[0]);
- FileJournalManager jm = new FileJournalManager(sd);
+ FileJournalManager jm = new FileJournalManager(sd, storage);
assertEquals(10*TXNS_PER_ROLL+1,
- jm.getNumberOfTransactions(1));
+ jm.getNumberOfTransactions(1, true));
}
@Test
@@ -288,14 +320,15 @@ public class TestFileJournalManager {
NNStorage.getInProgressEditsFileName(201),
NNStorage.getFinalizedEditsFileName(1001, 1100));
- FileJournalManager fjm = new FileJournalManager(sd);
+ // passing null for NNStorage because this unit test will not use it
+ FileJournalManager fjm = new FileJournalManager(sd, null);
assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1));
assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101));
assertEquals("[1001,1100]", getLogsAsString(fjm, 201));
try {
assertEquals("[]", getLogsAsString(fjm, 150));
fail("Did not throw when asking for a txn in the middle of a log");
- } catch (IOException ioe) {
+ } catch (IllegalStateException ioe) {
GenericTestUtils.assertExceptionContains(
"150 which is in the middle", ioe);
}
@@ -303,6 +336,60 @@ public class TestFileJournalManager {
"", getLogsAsString(fjm, 9999));
}
+ /**
+ * tests that passing an invalid dir to matchEditLogs throws IOException
+ */
+ @Test(expected = IOException.class)
+ public void testMatchEditLogInvalidDirThrowsIOException() throws IOException {
+ File badDir = new File("does not exist");
+ FileJournalManager.matchEditLogs(badDir);
+ }
+
+ /**
+ * Make sure that we starting reading the correct op when we request a stream
+ * with a txid in the middle of an edit log file.
+ */
+ @Test
+ public void testReadFromMiddleOfEditLog() throws CorruptionException,
+ IOException {
+ File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
+ NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
+ 10);
+ StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
+
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+
+ EditLogInputStream elis = jm.getInputStream(5, true);
+ FSEditLogOp op = elis.readOp();
+ assertEquals("read unexpected op", op.getTransactionId(), 5);
+ }
+
+ /**
+ * Make sure that in-progress streams aren't counted if we don't ask for
+ * them.
+ */
+ @Test
+ public void testExcludeInProgressStreams() throws CorruptionException,
+ IOException {
+ File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
+
+ // Don't close the edit log once the files have been set up.
+ NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
+ 10, false);
+ StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
+
+ FileJournalManager jm = new FileJournalManager(sd, storage);
+
+ // If we exclude the in-progess stream, we should only have 100 tx.
+ assertEquals(100, jm.getNumberOfTransactions(1, false));
+
+ EditLogInputStream elis = jm.getInputStream(90, false);
+ FSEditLogOp lastReadOp = null;
+ while ((lastReadOp = elis.readOp()) != null) {
+ assertTrue(lastReadOp.getTransactionId() <= 100);
+ }
+ }
+
private static String getLogsAsString(
FileJournalManager fjm, long firstTxId) throws IOException {
return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId));
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Sat Mar 3 00:42:49 2012
@@ -144,13 +144,13 @@ public class TestGenericJournalConf {
}
@Override
- public EditLogInputStream getInputStream(long fromTxnId)
+ public EditLogInputStream getInputStream(long fromTxnId, boolean inProgressOk)
throws IOException {
return null;
}
@Override
- public long getNumberOfTransactions(long fromTxnId)
+ public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
throws IOException {
return 0;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java Sat Mar 3 00:42:49 2012
@@ -61,6 +61,7 @@ public class TestNNStorageRetentionFunct
throws IOException {
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);
File sd0 = new File(TEST_ROOT_DIR, "nn0");
File sd1 = new File(TEST_ROOT_DIR, "nn1");
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Sat Mar 3 00:42:49 2012
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile;
@@ -33,6 +34,7 @@ import static org.apache.hadoop.hdfs.ser
import org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
@@ -46,6 +48,17 @@ import com.google.common.collect.Sets;
public class TestNNStorageRetentionManager {
+ Configuration conf = new Configuration();
+
+ /**
+ * For the purpose of this test, purge as many edits as we can
+ * with no extra "safety cushion"
+ */
+ @Before
+ public void setNoExtraEditRetention() {
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);
+ }
+
/**
* Test the "easy case" where we have more images in the
* directory than we need to keep. Should purge the
@@ -163,9 +176,27 @@ public class TestNNStorageRetentionManag
runTest(tc);
}
- private void runTest(TestCaseDescription tc) throws IOException {
- Configuration conf = new Configuration();
+ @Test
+ public void testRetainExtraLogs() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
+ 50);
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addRoot("/foo1", NameNodeDirType.IMAGE);
+ tc.addRoot("/foo2", NameNodeDirType.EDITS);
+ tc.addImage("/foo1/current/" + getImageFileName(100), true);
+ tc.addImage("/foo1/current/" + getImageFileName(200), true);
+ tc.addImage("/foo1/current/" + getImageFileName(300), false);
+ tc.addImage("/foo1/current/" + getImageFileName(400), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 200), true);
+ // Since we need 50 extra edits, *do* retain the 201-300 segment
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 300), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
+ runTest(tc);
+ }
+
+ private void runTest(TestCaseDescription tc) throws IOException {
StoragePurger mockPurger =
Mockito.mock(NNStorageRetentionManager.StoragePurger.class);
ArgumentCaptor<FSImageFile> imagesPurgedCaptor =
@@ -261,8 +292,9 @@ public class TestNNStorageRetentionManag
for (FakeRoot root : dirRoots.values()) {
if (!root.type.isOfType(NameNodeDirType.EDITS)) continue;
+ // passing null NNStorage for unit test because it does not use it
FileJournalManager fjm = new FileJournalManager(
- root.mockStorageDir());
+ root.mockStorageDir(), null);
fjm.purger = purger;
jms.add(fjm);
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourcePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourcePolicy.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourcePolicy.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourcePolicy.java Sat Mar 3 00:42:49 2012
@@ -50,13 +50,7 @@ public class TestNameNodeResourcePolicy
assertFalse(testResourceScenario(4, 0, 3, 0, 2));
assertTrue(testResourceScenario(4, 0, 3, 0, 1));
assertFalse(testResourceScenario(4, 0, 4, 0, 1));
- try {
- testResourceScenario(1, 0, 0, 0, 2);
- fail("Should fail if there are more minimum redundant resources than " +
- "total redundant resources");
- } catch (RuntimeException rte) {
- assertTrue(rte.getMessage().startsWith("Need a minimum"));
- }
+ assertFalse(testResourceScenario(1, 0, 0, 0, 2));
}
@Test
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Sat Mar 3 00:42:49 2012
@@ -24,6 +24,7 @@ import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -91,6 +92,9 @@ public class TestSecurityTokenEditLog ex
FileSystem fileSys = null;
try {
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
@@ -106,7 +110,6 @@ public class TestSecurityTokenEditLog ex
// set small size of flush buffer
editLog.setOutputBufferCapacity(2048);
- namesystem.getDelegationTokenSecretManager().startThreads();
// Create threads and make them run transactions concurrently.
Thread threadId[] = new Thread[NUM_THREADS];
@@ -141,7 +144,7 @@ public class TestSecurityTokenEditLog ex
System.out.println("Verifying file: " + editFile);
FSEditLogLoader loader = new FSEditLogLoader(namesystem);
- int numEdits = loader.loadFSEdits(
+ long numEdits = loader.loadFSEdits(
new EditLogFileInputStream(editFile), 1);
assertEquals("Verification for " + editFile, expectedTransactions, numEdits);
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java Sat Mar 3 00:42:49 2012
@@ -512,11 +512,10 @@ public class TestStartup extends TestCas
InetAddress inetAddress = InetAddress.getByAddress(b);
list.add(inetAddress.getHostName());
writeConfigFile(localFileSys, hostsFile, list);
- int numNameNodes = 1;
int numDatanodes = 1;
try {
- cluster = new MiniDFSCluster.Builder(conf).numNameNodes(numNameNodes)
+ cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes).setupHostsFile(true).build();
cluster.waitActive();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java Sat Mar 3 00:42:49 2012
@@ -19,10 +19,12 @@ package org.apache.hadoop.hdfs.server.na
import static org.junit.Assert.*;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -71,4 +73,25 @@ public class TestValidateConfigurationSe
DFSTestUtil.formatNameNode(conf);
NameNode nameNode = new NameNode(conf); // should be OK!
}
+
+ /**
+ * HDFS-3013: NameNode format command doesn't pick up
+ * dfs.namenode.name.dir.NameServiceId configuration.
+ */
+ @Test
+ public void testGenericKeysForNameNodeFormat()
+ throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ FileSystem.setDefaultUri(conf, "hdfs://localhost:8070");
+ conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1");
+ String nameDir = System.getProperty("java.io.tmpdir") + "/test.dfs.name";
+ File dir = new File(nameDir);
+ if (dir.exists()) {
+ FileUtil.fullyDelete(dir);
+ }
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + ".ns1", nameDir);
+ DFSTestUtil.formatNameNode(conf);
+ NameNode nameNode = new NameNode(conf);
+ FileUtil.fullyDelete(dir);
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java Sat Mar 3 00:42:49 2012
@@ -163,8 +163,13 @@ public class TestNameNodeMetrics {
// Corrupt first replica of the block
LocatedBlock block = NameNodeAdapter.getBlockLocations(
cluster.getNameNode(), file.toString(), 0, 1).get(0);
- bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
- "TEST");
+ cluster.getNamesystem().writeLock();
+ try {
+ bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
+ "TEST");
+ } finally {
+ cluster.getNamesystem().writeUnlock();
+ }
Thread.sleep(1000); // Wait for block to be marked corrupt
MetricsRecordBuilder rb = getMetrics(NS_METRICS);
assertGauge("CorruptBlocks", 1L, rb);
@@ -202,8 +207,13 @@ public class TestNameNodeMetrics {
// Corrupt the only replica of the block to result in a missing block
LocatedBlock block = NameNodeAdapter.getBlockLocations(
cluster.getNameNode(), file.toString(), 0, 1).get(0);
- bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
- "TEST");
+ cluster.getNamesystem().writeLock();
+ try {
+ bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
+ "TEST");
+ } finally {
+ cluster.getNamesystem().writeUnlock();
+ }
Thread.sleep(1000); // Wait for block to be marked corrupt
MetricsRecordBuilder rb = getMetrics(NS_METRICS);
assertGauge("UnderReplicatedBlocks", 1L, rb);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java Sat Mar 3 00:42:49 2012
@@ -24,6 +24,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.StringTokenizer;
import static org.junit.Assert.*;
@@ -32,6 +33,7 @@ import static org.apache.hadoop.fs.Commo
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.tools.GetConf;
import org.apache.hadoop.hdfs.tools.GetConf.Command;
@@ -72,7 +74,7 @@ public class TestGetConf {
String[] values = new String[nameServiceIdCount];
for (int i = 0; i < nameServiceIdCount; i++, portOffset++) {
String nsID = getNameServiceId(i);
- String specificKey = DFSUtil.getNameServiceIdKey(key, nsID);
+ String specificKey = DFSUtil.addKeySuffixes(key, nsID);
values[i] = "nn" + i + ":" + portOffset;
conf.set(specificKey, values[i]);
}
@@ -80,13 +82,13 @@ public class TestGetConf {
}
/*
- * Convert list of InetSocketAddress to string array with each address
- * represented as "host:port"
+ * Convert the map returned from DFSUtil functions to an array of
+ * addresses represented as "host:port"
*/
- private String[] toStringArray(List<InetSocketAddress> list) {
+ private String[] toStringArray(List<ConfiguredNNAddress> list) {
String[] ret = new String[list.size()];
for (int i = 0; i < list.size(); i++) {
- ret[i] = NetUtils.getHostPortString(list.get(i));
+ ret[i] = NetUtils.getHostPortString(list.get(i).getAddress());
}
return ret;
}
@@ -94,8 +96,8 @@ public class TestGetConf {
/**
* Using DFSUtil methods get the list of given {@code type} of address
*/
- private List<InetSocketAddress> getAddressListFromConf(TestType type,
- HdfsConfiguration conf) throws IOException {
+ private Map<String, Map<String, InetSocketAddress>> getAddressListFromConf(
+ TestType type, HdfsConfiguration conf) throws IOException {
switch (type) {
case NAMENODE:
return DFSUtil.getNNServiceRpcAddresses(conf);
@@ -161,7 +163,7 @@ public class TestGetConf {
* @param expected, expected addresses
*/
private void getAddressListFromTool(TestType type, HdfsConfiguration conf,
- boolean checkPort, List<InetSocketAddress> expected) throws Exception {
+ boolean checkPort, List<ConfiguredNNAddress> expected) throws Exception {
String out = getAddressListFromTool(type, conf, expected.size() != 0);
List<String> values = new ArrayList<String>();
@@ -176,7 +178,8 @@ public class TestGetConf {
// Convert expected list to String[] of hosts
int i = 0;
String[] expectedHosts = new String[expected.size()];
- for (InetSocketAddress addr : expected) {
+ for (ConfiguredNNAddress cnn : expected) {
+ InetSocketAddress addr = cnn.getAddress();
if (!checkPort) {
expectedHosts[i++] = addr.getHostName();
}else {
@@ -191,7 +194,9 @@ public class TestGetConf {
private void verifyAddresses(HdfsConfiguration conf, TestType type,
boolean checkPort, String... expected) throws Exception {
// Ensure DFSUtil returned the right set of addresses
- List<InetSocketAddress> list = getAddressListFromConf(type, conf);
+ Map<String, Map<String, InetSocketAddress>> map =
+ getAddressListFromConf(type, conf);
+ List<ConfiguredNNAddress> list = DFSUtil.flattenAddressMap(map);
String[] actual = toStringArray(list);
Arrays.sort(actual);
Arrays.sort(expected);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java Sat Mar 3 00:42:49 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
@@ -79,7 +80,7 @@ public class TestWebHdfsWithMultipleName
conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
cluster = new MiniDFSCluster.Builder(conf)
- .numNameNodes(nNameNodes)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
.numDataNodes(nDataNodes)
.build();
cluster.waitActive();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/GenericTestUtils.java?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/GenericTestUtils.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/test/GenericTestUtils.java Sat Mar 3 00:42:49 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
@@ -79,8 +80,8 @@ public abstract class GenericTestUtils {
public static void assertExceptionContains(String string, Throwable t) {
String msg = t.getMessage();
Assert.assertTrue(
- "Unexpected exception:" + StringUtils.stringifyException(t),
- msg.contains(string));
+ "Expected to find '" + string + "' but got unexpected exception:"
+ + StringUtils.stringifyException(t), msg.contains(string));
}
public static void waitFor(Supplier<Boolean> check,
@@ -109,7 +110,11 @@ public abstract class GenericTestUtils {
private final CountDownLatch fireLatch = new CountDownLatch(1);
private final CountDownLatch waitLatch = new CountDownLatch(1);
-
+ private final CountDownLatch resultLatch = new CountDownLatch(1);
+
+ // Result fields set after proceed() is called.
+ private volatile Throwable thrown;
+ private volatile Object returnValue;
public DelayAnswer(Log log) {
this.LOG = log;
@@ -144,7 +149,40 @@ public abstract class GenericTestUtils {
}
protected Object passThrough(InvocationOnMock invocation) throws Throwable {
- return invocation.callRealMethod();
+ try {
+ Object ret = invocation.callRealMethod();
+ returnValue = ret;
+ return ret;
+ } catch (Throwable t) {
+ thrown = t;
+ throw t;
+ } finally {
+ resultLatch.countDown();
+ }
+ }
+
+ /**
+ * After calling proceed(), this will wait until the call has
+ * completed and a result has been returned to the caller.
+ */
+ public void waitForResult() throws InterruptedException {
+ resultLatch.await();
+ }
+
+ /**
+ * After the call has gone through, return any exception that
+ * was thrown, or null if no exception was thrown.
+ */
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ /**
+ * After the call has gone through, return the call's return value,
+ * or null in case it was void or an exception was thrown.
+ */
+ public Object getReturnValue() {
+ return returnValue;
}
}
@@ -176,4 +214,35 @@ public abstract class GenericTestUtils {
}
}
+ /**
+ * An Answer implementation which sleeps for a random number of milliseconds
+ * between 0 and a configurable value before delegating to the real
+ * implementation of the method. This can be useful for drawing out race
+ * conditions.
+ */
+ public static class SleepAnswer implements Answer<Object> {
+ private final int maxSleepTime;
+ private static Random r = new Random();
+
+ public SleepAnswer(int maxSleepTime) {
+ this.maxSleepTime = maxSleepTime;
+ }
+
+ @Override
+ public Object answer(InvocationOnMock invocation) throws Throwable {
+ boolean interrupted = false;
+ try {
+ Thread.sleep(r.nextInt(maxSleepTime));
+ } catch (InterruptedException ie) {
+ interrupted = true;
+ }
+ try {
+ return invocation.callRealMethod();
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1296534&r1=1296533&r2=1296534&view=diff
==============================================================================
Binary files - no diff available.