You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2014/01/03 08:27:01 UTC
svn commit: r1555021 [13/15] - in
/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs/
hadoop-hdfs/dev-support/ hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apach...
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Fri Jan 3 07:26:52 2014
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -25,6 +27,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
@@ -68,7 +71,15 @@ import org.mockito.invocation.Invocation
/**
* This test simulates a variety of situations when blocks are being
* intentionally corrupted, unexpectedly modified, and so on before a block
- * report is happening
+ * report is happening.
+ *
+ * For each test case it runs two variations:
+ * #1 - For a given DN, the first variation sends block reports for all
+ * storages in a single call to the NN.
+ * #2 - For a given DN, the second variation sends block reports for each
+ * storage in a separate call.
+ *
+ * The behavior should be the same in either variation.
*/
public class TestBlockReport {
public static final Log LOG = LogFactory.getLog(TestBlockReport.class);
@@ -88,7 +99,7 @@ public class TestBlockReport {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
- Random rand = new Random(RAND_LIMIT);
+ private static Random rand = new Random(RAND_LIMIT);
private static Configuration conf;
@@ -112,6 +123,155 @@ public class TestBlockReport {
cluster.shutdown();
}
+ // Generate a block report, optionally corrupting the generation
+ // stamp and/or length of one block.
+ private static StorageBlockReport[] getBlockReports(
+ DataNode dn, String bpid, boolean corruptOneBlockGs,
+ boolean corruptOneBlockLen) {
+ Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
+ dn.getFSDataset().getBlockReports(bpid);
+
+ // Send block report
+ StorageBlockReport[] reports =
+ new StorageBlockReport[perVolumeBlockLists.size()];
+ boolean corruptedGs = false;
+ boolean corruptedLen = false;
+
+ int reportIndex = 0;
+ for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
+ DatanodeStorage dnStorage = kvPair.getKey();
+ BlockListAsLongs blockList = kvPair.getValue();
+
+ // Walk the list of blocks until we find one each to corrupt the
+ // generation stamp and length, if so requested.
+ for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) {
+ if (corruptOneBlockGs && !corruptedGs) {
+ blockList.corruptBlockGSForTesting(i, rand);
+ LOG.info("Corrupted the GS for block ID " + i);
+ corruptedGs = true;
+ } else if (corruptOneBlockLen && !corruptedLen) {
+ blockList.corruptBlockLengthForTesting(i, rand);
+ LOG.info("Corrupted the length for block ID " + i);
+ corruptedLen = true;
+ } else {
+ break;
+ }
+ }
+
+ reports[reportIndex++] =
+ new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
+ }
+
+ return reports;
+ }
+
+ /**
+ * Utility routine to send block reports to the NN, either in a single call
+ * or reporting one storage per call.
+ *
+ * @param dnR
+ * @param poolId
+ * @param reports
+ * @param needtoSplit
+ * @throws IOException
+ */
+ private void sendBlockReports(DatanodeRegistration dnR, String poolId,
+ StorageBlockReport[] reports, boolean needtoSplit) throws IOException {
+ if (!needtoSplit) {
+ LOG.info("Sending combined block reports for " + dnR);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
+ } else {
+ for (StorageBlockReport report : reports) {
+ LOG.info("Sending block report for storage " + report.getStorage().getStorageID());
+ StorageBlockReport[] singletonReport = { report };
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, singletonReport);
+ }
+ }
+ }
+
+ /**
+ * Test variations blockReport_01 through blockReport_09 with combined
+ * and split block reports.
+ */
+ @Test
+ public void blockReportCombined_01() throws IOException {
+ blockReport_01(false);
+ }
+
+ @Test
+ public void blockReportSplit_01() throws IOException {
+ blockReport_01(true);
+ }
+
+ @Test
+ public void blockReportCombined_02() throws IOException {
+ blockReport_02(false);
+ }
+
+ @Test
+ public void blockReportSplit_02() throws IOException {
+ blockReport_02(true);
+ }
+
+ @Test
+ public void blockReportCombined_03() throws IOException {
+ blockReport_03(false);
+ }
+
+ @Test
+ public void blockReportSplit_03() throws IOException {
+ blockReport_03(true);
+ }
+
+ @Test
+ public void blockReportCombined_04() throws IOException {
+ blockReport_04(false);
+ }
+
+ @Test
+ public void blockReportSplit_04() throws IOException {
+ blockReport_04(true);
+ }
+
+ @Test
+ public void blockReportCombined_06() throws Exception {
+ blockReport_06(false);
+ }
+
+ @Test
+ public void blockReportSplit_06() throws Exception {
+ blockReport_06(true);
+ }
+
+ @Test
+ public void blockReportCombined_07() throws Exception {
+ blockReport_07(false);
+ }
+
+ @Test
+ public void blockReportSplit_07() throws Exception {
+ blockReport_07(true);
+ }
+
+ @Test
+ public void blockReportCombined_08() throws Exception {
+ blockReport_08(false);
+ }
+
+ @Test
+ public void blockReportSplit_08() throws Exception {
+ blockReport_08(true);
+ }
+
+ @Test
+ public void blockReportCombined_09() throws Exception {
+ blockReport_09(false);
+ }
+
+ @Test
+ public void blockReportSplit_09() throws Exception {
+ blockReport_09(true);
+ }
/**
* Test write a file, verifies and closes it. Then the length of the blocks
* are messed up and BlockReport is forced.
@@ -119,8 +279,7 @@ public class TestBlockReport {
*
* @throws java.io.IOException on an error
*/
- @Test
- public void blockReport_01() throws IOException {
+ private void blockReport_01(boolean splitBlockReports) throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
@@ -152,10 +311,8 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N0);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
List<LocatedBlock> blocksAfterReport =
DFSTestUtil.getAllBlocks(fs.open(filePath));
@@ -181,8 +338,7 @@ public class TestBlockReport {
*
* @throws IOException in case of errors
*/
- @Test
- public void blockReport_02() throws IOException {
+ private void blockReport_02(boolean splitBlockReports) throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
LOG.info("Running test " + METHOD_NAME);
@@ -210,7 +366,6 @@ public class TestBlockReport {
for (Integer aRemovedIndex : removedIndex) {
blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock());
}
- ArrayList<Block> blocks = locatedToBlocks(lBlocks, removedIndex);
if(LOG.isDebugEnabled()) {
LOG.debug("Number of blocks allocated " + lBlocks.size());
@@ -224,8 +379,11 @@ public class TestBlockReport {
for (File f : findAllFiles(dataDir,
new MyFileFilter(b.getBlockName(), true))) {
DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b);
- if (!f.delete())
+ if (!f.delete()) {
LOG.warn("Couldn't delete " + b.getBlockName());
+ } else {
+ LOG.debug("Deleted file " + f.toString());
+ }
}
}
@@ -234,10 +392,8 @@ public class TestBlockReport {
// all blocks belong to the same file, hence same BP
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem()
.getBlockManager());
@@ -252,61 +408,64 @@ public class TestBlockReport {
/**
- * Test writes a file and closes it. Then test finds a block
- * and changes its GS to be < of original one.
- * New empty block is added to the list of blocks.
+ * Test writes a file and closes it.
+ * Block reported is generated with a bad GS for a single block.
* Block report is forced and the check for # of corrupted blocks is performed.
*
* @throws IOException in case of an error
*/
- @Test
- public void blockReport_03() throws IOException {
+ private void blockReport_03(boolean splitBlockReports) throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
-
- ArrayList<Block> blocks =
- prepareForRide(filePath, METHOD_NAME, FILE_SIZE);
-
- // The block with modified GS won't be found. Has to be deleted
- blocks.get(0).setGenerationStamp(rand.nextLong());
- // This new block is unknown to NN and will be mark for deletion.
- blocks.add(new Block());
+ ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
// all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N0);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- DatanodeCommand dnCmd =
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Got the command: " + dnCmd);
- }
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
- assertEquals("Wrong number of CorruptedReplica+PendingDeletion " +
- "blocks is found", 2,
- cluster.getNamesystem().getCorruptReplicaBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
}
/**
- * This test isn't a representative case for BlockReport
- * The empty method is going to be left here to keep the naming
- * of the test plan in synch with the actual implementation
+ * Test writes a file and closes it.
+ * Block reported is generated with an extra block.
+ * Block report is forced and the check for # of pendingdeletion
+ * blocks is performed.
+ *
+ * @throws IOException in case of an error
*/
- public void blockReport_04() {
- }
+ private void blockReport_04(boolean splitBlockReports) throws IOException {
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path filePath = new Path("/" + METHOD_NAME + ".dat");
+ DFSTestUtil.createFile(fs, filePath,
+ FILE_SIZE, REPL_FACTOR, rand.nextLong());
+
+
+ DataNode dn = cluster.getDataNodes().get(DN_N0);
+ // all blocks belong to the same file, hence same BP
+ String poolId = cluster.getNamesystem().getBlockPoolId();
+
+ // Create a bogus new block which will not be present on the namenode.
+ ExtendedBlock b = new ExtendedBlock(
+ poolId, rand.nextLong(), 1024L, rand.nextLong());
+ dn.getFSDataset().createRbw(b);
+
+ DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
+ printStats();
- // Client requests new block from NN. The test corrupts this very block
- // and forces new block report.
- // The test case isn't specific for BlockReport because it relies on
- // BlockScanner which is out of scope of this test
- // Keeping the name to be in synch with the test plan
- //
- public void blockReport_05() {
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(1L));
}
/**
@@ -317,23 +476,20 @@ public class TestBlockReport {
*
* @throws IOException in case of an error
*/
- @Test
- public void blockReport_06() throws Exception {
+ private void blockReport_06(boolean splitBlockReports) throws Exception {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
- ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
+ writeFile(METHOD_NAME, FILE_SIZE, filePath);
startDNandWait(filePath, true);
- // all blocks belong to the same file, hence same BP
+ // all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
assertEquals("Wrong number of PendingReplication Blocks",
0, cluster.getNamesystem().getUnderReplicatedBlocks());
@@ -352,69 +508,40 @@ public class TestBlockReport {
*
* @throws IOException in case of an error
*/
- @Test
- // Currently this test is failing as expected 'cause the correct behavior is
- // not yet implemented (9/15/09)
- public void blockReport_07() throws Exception {
+ private void blockReport_07(boolean splitBlockReports) throws Exception {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
// write file and start second node to be "older" than the original
- ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
+ writeFile(METHOD_NAME, FILE_SIZE, filePath);
startDNandWait(filePath, true);
- int randIndex = rand.nextInt(blocks.size());
- // Get a block and screw its GS
- Block corruptedBlock = blocks.get(randIndex);
- String secondNode = cluster.getDataNodes().get(DN_N1).getStorageId();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Working with " + secondNode);
- LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp());
- }
- corruptBlockGS(corruptedBlock);
- if(LOG.isDebugEnabled()) {
- LOG.debug("BlockGS after " + blocks.get(randIndex).getGenerationStamp());
- LOG.debug("Done corrupting GS of " + corruptedBlock.getBlockName());
- }
// all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
- assertEquals("Wrong number of Corrupted blocks",
- 1, cluster.getNamesystem().getCorruptReplicaBlocks() +
-// the following might have to be added into the equation if
-// the same block could be in two different states at the same time
-// and then the expected number of has to be changed to '2'
-// cluster.getNamesystem().getPendingReplicationBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
-
- // Get another block and screw its length to be less than original
- if (randIndex == 0)
- randIndex++;
- else
- randIndex--;
- corruptedBlock = blocks.get(randIndex);
- corruptBlockLen(corruptedBlock);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Done corrupting length of " + corruptedBlock.getBlockName());
- }
-
- report[0] = new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs());
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
+ assertThat("Wrong number of PendingReplication blocks",
+ cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
+
+ reports = getBlockReports(dn, poolId, true, true);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
- assertEquals("Wrong number of Corrupted blocks",
- 2, cluster.getNamesystem().getCorruptReplicaBlocks() +
- cluster.getNamesystem().getPendingReplicationBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
+ assertThat("Wrong number of PendingReplication blocks",
+ cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
printStats();
@@ -432,8 +559,7 @@ public class TestBlockReport {
*
* @throws IOException in case of an error
*/
- @Test
- public void blockReport_08() throws IOException {
+ private void blockReport_08(boolean splitBlockReports) throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
@@ -457,10 +583,8 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
assertEquals("Wrong number of PendingReplication blocks",
blocks.size(), cluster.getNamesystem().getPendingReplicationBlocks());
@@ -476,8 +600,7 @@ public class TestBlockReport {
// Similar to BlockReport_08 but corrupts GS and len of the TEMPORARY's
// replica block. Expect the same behaviour: NN should simply ignore this
// block
- @Test
- public void blockReport_09() throws IOException {
+ private void blockReport_09(boolean splitBlockReports) throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
@@ -490,14 +613,11 @@ public class TestBlockReport {
// write file and start second node to be "older" than the original
try {
- ArrayList<Block> blocks =
- writeFile(METHOD_NAME, 12 * bytesChkSum, filePath);
+ writeFile(METHOD_NAME, 12 * bytesChkSum, filePath);
Block bl = findBlock(filePath, 12 * bytesChkSum);
BlockChecker bc = new BlockChecker(filePath);
bc.start();
- corruptBlockGS(bl);
- corruptBlockLen(bl);
waitForTempReplica(bl, DN_N1);
@@ -505,10 +625,8 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, true, true);
+ sendBlockReports(dnR, poolId, reports, splitBlockReports);
printStats();
assertEquals("Wrong number of PendingReplication blocks",
2, cluster.getNamesystem().getPendingReplicationBlocks());
@@ -783,38 +901,6 @@ public class TestBlockReport {
((Log4JLogger) TestBlockReport.LOG).getLogger().setLevel(Level.ALL);
}
- private void corruptBlockLen(final Block block)
- throws IOException {
- if (block == null) {
- throw new IOException("Block isn't suppose to be null");
- }
- long oldLen = block.getNumBytes();
- long newLen = oldLen - rand.nextLong();
- assertTrue("Old and new length shouldn't be the same",
- block.getNumBytes() != newLen);
- block.setNumBytes(newLen);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Length of " + block.getBlockName() +
- " is changed to " + newLen + " from " + oldLen);
- }
- }
-
- private void corruptBlockGS(final Block block)
- throws IOException {
- if (block == null) {
- throw new IOException("Block isn't suppose to be null");
- }
- long oldGS = block.getGenerationStamp();
- long newGS = oldGS - rand.nextLong();
- assertTrue("Old and new GS shouldn't be the same",
- block.getGenerationStamp() != newGS);
- block.setGenerationStamp(newGS);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Generation stamp of " + block.getBlockName() +
- " is changed to " + block.getGenerationStamp() + " from " + oldGS);
- }
- }
-
private Block findBlock(Path path, long size) throws IOException {
Block ret;
List<LocatedBlock> lbs =
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java Fri Jan 3 07:26:52 2014
@@ -19,12 +19,14 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.*;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.StorageType;
import org.junit.Test;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
@@ -34,19 +36,69 @@ import org.apache.hadoop.hdfs.server.dat
public class TestDataDirs {
- @Test (timeout = 10000)
- public void testGetDataDirsFromURIs() throws Throwable {
+ @Test (timeout = 30000)
+ public void testDataDirParsing() throws Throwable {
+ Configuration conf = new Configuration();
+ List<StorageLocation> locations;
+ File dir0 = new File("/dir0");
+ File dir1 = new File("/dir1");
+ File dir2 = new File("/dir2");
+ File dir3 = new File("/dir3");
+
+ // Verify that a valid string is correctly parsed, and that storage
+ // type is not case-sensitive
+ String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1);
+ locations = DataNode.getStorageLocations(conf);
+ assertThat(locations.size(), is(4));
+ assertThat(locations.get(0).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(0).getUri(), is(dir0.toURI()));
+ assertThat(locations.get(1).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(1).getUri(), is(dir1.toURI()));
+ assertThat(locations.get(2).getStorageType(), is(StorageType.SSD));
+ assertThat(locations.get(2).getUri(), is(dir2.toURI()));
+ assertThat(locations.get(3).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(3).getUri(), is(dir3.toURI()));
+
+ // Verify that an unrecognized storage type result in an exception.
+ String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2);
+ try {
+ locations = DataNode.getStorageLocations(conf);
+ fail();
+ } catch(IllegalArgumentException iae) {
+ DataNode.LOG.info("The exception is expected.", iae);
+ }
+
+ // Assert that a string with no storage type specified is
+ // correctly parsed and the default storage type is picked up.
+ String locations3 = "/dir0,/dir1";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3);
+ locations = DataNode.getStorageLocations(conf);
+ assertThat(locations.size(), is(2));
+ assertThat(locations.get(0).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(0).getUri(), is(dir0.toURI()));
+ assertThat(locations.get(1).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(1).getUri(), is(dir1.toURI()));
+ }
+
+ @Test (timeout = 30000)
+ public void testDataDirValidation() throws Throwable {
DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class);
doThrow(new IOException()).doThrow(new IOException()).doNothing()
.when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class));
LocalFileSystem fs = mock(LocalFileSystem.class);
- Collection<URI> uris = Arrays.asList(new URI("file:/p1/"),
- new URI("file:/p2/"), new URI("file:/p3/"));
+ AbstractList<StorageLocation> locations = new ArrayList<StorageLocation>();
- List<File> dirs = DataNode.getDataDirsFromURIs(uris, fs, diskChecker);
- assertEquals("number of valid data dirs", 1, dirs.size());
- String validDir = dirs.iterator().next().getPath();
- assertEquals("p3 should be valid", new File("/p3").getPath(), validDir);
+ locations.add(StorageLocation.parse("file:/p1/"));
+ locations.add(StorageLocation.parse("file:/p2/"));
+ locations.add(StorageLocation.parse("file:/p3/"));
+
+ List<StorageLocation> checkedLocations =
+ DataNode.checkStorageLocations(locations, fs, diskChecker);
+ assertEquals("number of valid data dirs", 1, checkedLocations.size());
+ String validDir = checkedLocations.iterator().next().getFile().getPath();
+ assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir));
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Fri Jan 3 07:26:52 2014
@@ -163,7 +163,7 @@ public class TestDataNodeMultipleRegistr
for (BPOfferService bpos : dn.getAllBpOs()) {
LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid="
- + bpos.bpRegistration.getStorageID() + "; nna=" +
+ + bpos.bpRegistration.getDatanodeUuid() + "; nna=" +
getNNSocketAddress(bpos));
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Fri Jan 3 07:26:52 2014
@@ -42,11 +42,13 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -151,13 +153,23 @@ public class TestDataNodeVolumeFailure {
DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
String bpid = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
- final StorageBlockReport[] report = {
- new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid
- ).getBlockListAsLongs())
- };
- cluster.getNameNodeRpc().blockReport(dnR, bpid, report);
+
+ Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
+ dn.getFSDataset().getBlockReports(bpid);
+
+ // Send block report
+ StorageBlockReport[] reports =
+ new StorageBlockReport[perVolumeBlockLists.size()];
+
+ int reportIndex = 0;
+ for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
+ DatanodeStorage dnStorage = kvPair.getKey();
+ BlockListAsLongs blockList = kvPair.getValue();
+ reports[reportIndex++] =
+ new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
+ }
+
+ cluster.getNameNodeRpc().blockReport(dnR, bpid, reports);
// verify number of blocks and files...
verify(filename, filesize);
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Fri Jan 3 07:26:52 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -406,6 +407,16 @@ public class TestDirectoryScanner {
public File getFinalizedDir(String bpid) throws IOException {
return new File("/base/current/" + bpid + "/finalized");
}
+
+ @Override
+ public StorageType getStorageType() {
+ return StorageType.DEFAULT;
+ }
+
+ @Override
+ public String getStorageID() {
+ return "";
+ }
}
private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi();
@@ -436,7 +447,7 @@ public class TestDirectoryScanner {
void testScanInfoObject(long blockId) throws Exception {
DirectoryScanner.ScanInfo scanInfo =
- new DirectoryScanner.ScanInfo(blockId);
+ new DirectoryScanner.ScanInfo(blockId, null, null, null);
assertEquals(blockId, scanInfo.getBlockId());
assertNull(scanInfo.getBlockFile());
assertNull(scanInfo.getMetaFile());
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java Fri Jan 3 07:26:52 2014
@@ -36,6 +36,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.Path;
@@ -82,7 +83,11 @@ public class TestFsDatasetCache {
// Most Linux installs allow a default of 64KB locked memory
private static final long CACHE_CAPACITY = 64 * 1024;
- private static final long BLOCK_SIZE = 4096;
+ // mlock always locks the entire page. So we don't need to deal with this
+ // rounding, use the OS page size for the block size.
+ private static final long PAGE_SIZE =
+ NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize();
+ private static final long BLOCK_SIZE = PAGE_SIZE;
private static Configuration conf;
private static MiniDFSCluster cluster = null;
@@ -104,14 +109,12 @@ public class TestFsDatasetCache {
public void setUp() throws Exception {
assumeTrue(!Path.WINDOWS);
conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY, true);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS,
500);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
CACHE_CAPACITY);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY, true);
prevCacheManipulator = NativeIO.POSIX.getCacheManipulator();
NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
@@ -451,4 +454,27 @@ public class TestFsDatasetCache {
}
}, 100, 10000);
}
+
+ @Test(timeout=60000)
+ public void testPageRounder() throws Exception {
+ // Write a small file
+ Path fileName = new Path("/testPageRounder");
+ final int smallBlocks = 512; // This should be smaller than the page size
+ assertTrue("Page size should be greater than smallBlocks!",
+ PAGE_SIZE > smallBlocks);
+ final int numBlocks = 5;
+ final int fileLen = smallBlocks * numBlocks;
+ FSDataOutputStream out =
+ fs.create(fileName, false, 4096, (short)1, smallBlocks);
+ out.write(new byte[fileLen]);
+ out.close();
+ HdfsBlockLocation[] locs = (HdfsBlockLocation[])fs.getFileBlockLocations(
+ fileName, 0, fileLen);
+ // Cache the file and check the sizes match the page size
+ setHeartbeatResponse(cacheBlocks(locs));
+ verifyExpectedCacheUsage(PAGE_SIZE * numBlocks, numBlocks);
+ // Uncache and check that it decrements by the page size too
+ setHeartbeatResponse(uncacheBlocks(locs));
+ verifyExpectedCacheUsage(0, 0);
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Fri Jan 3 07:26:52 2014
@@ -311,7 +311,7 @@ public class TestSimulatedFSDataset {
}
private SimulatedFSDataset getSimulatedFSDataset() {
- SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf);
+ SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Fri Jan 3 07:26:52 2014
@@ -17,6 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -25,28 +33,40 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
-import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.util.*;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-
/**
* Main class for a series of name-node benchmarks.
*
@@ -586,6 +606,98 @@ public class NNThroughputBenchmark imple
}
/**
+ * Directory creation statistics.
+ *
+ * Each thread creates the same (+ or -1) number of directories.
+ * Directory names are pre-generated during initialization.
+ */
+ class MkdirsStats extends OperationStatsBase {
+ // Operation types
+ static final String OP_MKDIRS_NAME = "mkdirs";
+ static final String OP_MKDIRS_USAGE = "-op mkdirs [-threads T] [-dirs N] " +
+ "[-dirsPerDir P]";
+
+ protected FileNameGenerator nameGenerator;
+ protected String[][] dirPaths;
+
+ MkdirsStats(List<String> args) {
+ super();
+ parseArguments(args);
+ }
+
+ @Override
+ String getOpName() {
+ return OP_MKDIRS_NAME;
+ }
+
+ @Override
+ void parseArguments(List<String> args) {
+ boolean ignoreUnrelatedOptions = verifyOpArgument(args);
+ int nrDirsPerDir = 2;
+ for (int i = 2; i < args.size(); i++) { // parse command line
+ if(args.get(i).equals("-dirs")) {
+ if(i+1 == args.size()) printUsage();
+ numOpsRequired = Integer.parseInt(args.get(++i));
+ } else if(args.get(i).equals("-threads")) {
+ if(i+1 == args.size()) printUsage();
+ numThreads = Integer.parseInt(args.get(++i));
+ } else if(args.get(i).equals("-dirsPerDir")) {
+ if(i+1 == args.size()) printUsage();
+ nrDirsPerDir = Integer.parseInt(args.get(++i));
+ } else if(!ignoreUnrelatedOptions)
+ printUsage();
+ }
+ nameGenerator = new FileNameGenerator(getBaseDir(), nrDirsPerDir);
+ }
+
+ @Override
+ void generateInputs(int[] opsPerThread) throws IOException {
+ assert opsPerThread.length == numThreads : "Error opsPerThread.length";
+ nameNodeProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
+ false);
+ LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
+ dirPaths = new String[numThreads][];
+ for(int idx=0; idx < numThreads; idx++) {
+ int threadOps = opsPerThread[idx];
+ dirPaths[idx] = new String[threadOps];
+ for(int jdx=0; jdx < threadOps; jdx++)
+ dirPaths[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
+ }
+ }
+
+ /**
+ * returns client name
+ */
+ @Override
+ String getExecutionArgument(int daemonId) {
+ return getClientName(daemonId);
+ }
+
+ /**
+ * Do mkdirs operation.
+ */
+ @Override
+ long executeOp(int daemonId, int inputIdx, String clientName)
+ throws IOException {
+ long start = Time.now();
+ nameNodeProto.mkdirs(dirPaths[daemonId][inputIdx],
+ FsPermission.getDefault(), true);
+ long end = Time.now();
+ return end-start;
+ }
+
+ @Override
+ void printResults() {
+ LOG.info("--- " + getOpName() + " inputs ---");
+ LOG.info("nrDirs = " + numOpsRequired);
+ LOG.info("nrThreads = " + numThreads);
+ LOG.info("nrDirsPerDir = " + nameGenerator.getFilesPerDirectory());
+ printStats();
+ }
+ }
+
+ /**
* Open file statistics.
*
* Measure how many open calls (getBlockLocations())
@@ -817,17 +929,16 @@ public class NNThroughputBenchmark imple
dnRegistration = new DatanodeRegistration(
new DatanodeID(DNS.getDefaultIP("default"),
DNS.getDefaultHost("default", "default"),
- "", getNodePort(dnIdx),
+ DataNode.generateUuid(), getNodePort(dnIdx),
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT),
- new DataStorage(nsInfo, ""),
+ new DataStorage(nsInfo),
new ExportedBlockKeys(), VersionInfo.getVersion());
- DataNode.setNewStorageID(dnRegistration);
// register datanode
dnRegistration = nameNodeProto.registerDatanode(dnRegistration);
//first block reports
- storage = new DatanodeStorage(dnRegistration.getStorageID());
+ storage = new DatanodeStorage(dnRegistration.getDatanodeUuid());
final StorageBlockReport[] reports = {
new StorageBlockReport(storage,
new BlockListAsLongs(null, null).getBlockListAsLongs())
@@ -843,7 +954,7 @@ public class NNThroughputBenchmark imple
void sendHeartbeat() throws IOException {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
- StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
+ StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0).getCommands();
@@ -890,7 +1001,7 @@ public class NNThroughputBenchmark imple
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
// register datanode
- StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
+ StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0).getCommands();
@@ -920,14 +1031,14 @@ public class NNThroughputBenchmark imple
DatanodeInfo dnInfo = blockTargets[t];
DatanodeRegistration receivedDNReg;
receivedDNReg = new DatanodeRegistration(dnInfo,
- new DataStorage(nsInfo, dnInfo.getStorageID()),
+ new DataStorage(nsInfo),
new ExportedBlockKeys(), VersionInfo.getVersion());
ReceivedDeletedBlockInfo[] rdBlocks = {
new ReceivedDeletedBlockInfo(
blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- receivedDNReg.getStorageID(), rdBlocks) };
+ receivedDNReg.getDatanodeUuid(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode
.getNamesystem().getBlockPoolId(), report);
}
@@ -1050,7 +1161,7 @@ public class NNThroughputBenchmark imple
loc.getBlock().getLocalBlock(),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) };
+ datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc
.getBlock().getBlockPoolId(), report);
}
@@ -1260,6 +1371,7 @@ public class NNThroughputBenchmark imple
System.err.println("Usage: NNThroughputBenchmark"
+ "\n\t" + OperationStatsBase.OP_ALL_USAGE
+ " | \n\t" + CreateFileStats.OP_CREATE_USAGE
+ + " | \n\t" + MkdirsStats.OP_MKDIRS_USAGE
+ " | \n\t" + OpenFileStats.OP_OPEN_USAGE
+ " | \n\t" + DeleteFileStats.OP_DELETE_USAGE
+ " | \n\t" + FileStatusStats.OP_FILE_STATUS_USAGE
@@ -1309,6 +1421,10 @@ public class NNThroughputBenchmark imple
opStat = new CreateFileStats(args);
ops.add(opStat);
}
+ if(runAll || MkdirsStats.OP_MKDIRS_NAME.equals(type)) {
+ opStat = new MkdirsStats(args);
+ ops.add(opStat);
+ }
if(runAll || OpenFileStats.OP_OPEN_NAME.equals(type)) {
opStat = new OpenFileStats(args);
ops.add(opStat);
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Fri Jan 3 07:26:52 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp;
@@ -111,8 +112,8 @@ public class NameNodeAdapter {
public static HeartbeatResponse sendHeartBeat(DatanodeRegistration nodeReg,
DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException {
- return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(),
- dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(),
+ return namesystem.handleHeartbeat(nodeReg,
+ BlockManagerTestUtil.getStorageReportsForDatanode(dd),
dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0);
}
@@ -238,3 +239,4 @@ public class NameNodeAdapter {
return NNStorage.getInProgressEditsFile(sd, startTxId);
}
}
+
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Fri Jan 3 07:26:52 2014
@@ -239,7 +239,7 @@ public class OfflineEditsViewerHelper {
.setOwnerName("carlton")
.setGroupName("party")
.setMode(new FsPermission((short)0700))
- .setWeight(1989));
+ .setLimit(1989l));
// OP_ADD_PATH_BASED_CACHE_DIRECTIVE 33
long id = dfs.addCacheDirective(
new CacheDirectiveInfo.Builder().
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1555021&r1=1555020&r2=1555021&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Fri Jan 3 07:26:52 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.net.Node;
@@ -99,13 +100,13 @@ public class TestAddBlockRetry {
bmField.setAccessible(true);
bmField.set(ns, spyBM);
- doAnswer(new Answer<DatanodeDescriptor[]>() {
+ doAnswer(new Answer<DatanodeStorageInfo[]>() {
@Override
- public DatanodeDescriptor[] answer(InvocationOnMock invocation)
+ public DatanodeStorageInfo[] answer(InvocationOnMock invocation)
throws Throwable {
LOG.info("chooseTarget for " + src);
- DatanodeDescriptor[] ret =
- (DatanodeDescriptor[]) invocation.callRealMethod();
+ DatanodeStorageInfo[] ret =
+ (DatanodeStorageInfo[]) invocation.callRealMethod();
count++;
if(count == 1) { // run second addBlock()
LOG.info("Starting second addBlock for " + src);