You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/22 23:11:24 UTC
[30/50] [abbrv] hadoop git commit: HDFS-11117. Refactor striped file
tests to allow flexibly test erasure coding policy. Contributed by Sammi Chen
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index b532443..1574a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -39,23 +36,18 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -90,10 +82,14 @@ public class TestDecommissionWithStriped {
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem dfs;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
private int numDNs;
- private final int blockSize = StripedFileTestUtil.blockSize;
- private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int blockSize = cellSize * 4;
+ private final int blockGroupSize = blockSize * dataBlocks;
private final Path ecDir = new Path("/" + this.getClass().getSimpleName());
private FSNamesystem fsn;
@@ -132,12 +128,12 @@ public class TestDecommissionWithStriped {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(
DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
- StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE - 1);
+ cellSize - 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
false);
- numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2;
+ numDNs = dataBlocks + parityBlocks + 2;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
dfs = cluster.getFileSystem(0);
@@ -194,7 +190,7 @@ public class TestDecommissionWithStriped {
LOG.info("Starting test testDecommissionWithURBlocksForSameBlockGroup");
final Path ecFile = new Path(ecDir, "testDecommissionWithCorruptBlocks");
- int writeBytes = BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2;
+ int writeBytes = cellSize * dataBlocks * 2;
writeStripedFile(dfs, ecFile, writeBytes);
Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
@@ -202,8 +198,8 @@ public class TestDecommissionWithStriped {
LocatedBlock lb = dfs.getClient().getLocatedBlocks(ecFile.toString(), 0)
.get(0);
DatanodeInfo[] dnLocs = lb.getLocations();
- assertEquals(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS, dnLocs.length);
- int decommNodeIndex = NUM_DATA_BLOCKS - 1;
+ assertEquals(dataBlocks + parityBlocks, dnLocs.length);
+ int decommNodeIndex = dataBlocks - 1;
int stopNodeIndex = 1;
// add the nodes which will be decommissioning
@@ -273,7 +269,7 @@ public class TestDecommissionWithStriped {
assertNull(checkFile(dfs, ecFile, 9, decommisionNodes, numDNs));
StripedFileTestUtil.checkData(dfs, ecFile, writeBytes, decommisionNodes,
- null);
+ null, blockGroupSize);
cleanupFile(dfs, ecFile);
}
@@ -294,7 +290,7 @@ public class TestDecommissionWithStriped {
LOG.info("Starting test testFileChecksumAfterDecommission");
final Path ecFile = new Path(ecDir, "testFileChecksumAfterDecommission");
- int writeBytes = BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS;
+ int writeBytes = cellSize * dataBlocks;
writeStripedFile(dfs, ecFile, writeBytes);
Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
FileChecksum fileChecksum1 = dfs.getFileChecksum(ecFile, writeBytes);
@@ -303,7 +299,7 @@ public class TestDecommissionWithStriped {
LocatedBlock lb = dfs.getClient().getLocatedBlocks(ecFile.toString(), 0)
.get(0);
DatanodeInfo[] dnLocs = lb.getLocations();
- assertEquals(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS, dnLocs.length);
+ assertEquals(dataBlocks + parityBlocks, dnLocs.length);
int decommNodeIndex = 1;
// add the node which will be decommissioning
@@ -312,7 +308,7 @@ public class TestDecommissionWithStriped {
assertEquals(decommisionNodes.size(), fsn.getNumDecomLiveDataNodes());
assertNull(checkFile(dfs, ecFile, 9, decommisionNodes, numDNs));
StripedFileTestUtil.checkData(dfs, ecFile, writeBytes, decommisionNodes,
- null);
+ null, blockGroupSize);
// verify checksum
FileChecksum fileChecksum2 = dfs.getFileChecksum(ecFile, writeBytes);
@@ -355,7 +351,7 @@ public class TestDecommissionWithStriped {
assertNull(checkFile(dfs, ecFile, storageCount, decommisionNodes, numDNs));
StripedFileTestUtil.checkData(dfs, ecFile, writeBytes, decommisionNodes,
- null);
+ null, blockGroupSize);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
@@ -437,7 +433,7 @@ public class TestDecommissionWithStriped {
StripedFileTestUtil.waitBlockGroupsReported(dfs, ecFile.toString());
StripedFileTestUtil.checkData(dfs, ecFile, writeBytes,
- new ArrayList<DatanodeInfo>(), null);
+ new ArrayList<DatanodeInfo>(), null, blockGroupSize);
}
private void writeConfigFile(Path name, List<String> nodes)
@@ -528,7 +524,7 @@ public class TestDecommissionWithStriped {
* decommissioned nodes, verify their replication is equal to what is
* specified.
*
- * @param downnode
+ * @param decommissionedNodes
* - if null, there is no decommissioned node for this file.
* @return - null if no failure found, else an error message string.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
index 5b63cc4..e02c0bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
@@ -39,16 +39,17 @@ public class TestErasureCodingPolicyWithSnapshot {
private DistributedFileSystem fs;
private Configuration conf;
- private final static short GROUP_SIZE = (short) (StripedFileTestUtil.
- NUM_DATA_BLOCKS + StripedFileTestUtil.NUM_PARITY_BLOCKS);
private final static int SUCCESS = 0;
private final ErasureCodingPolicy sysDefaultPolicy =
- StripedFileTestUtil.TEST_EC_POLICY;
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final short groupSize = (short) (
+ sysDefaultPolicy.getNumDataUnits() +
+ sysDefaultPolicy.getNumParityUnits());
@Before
public void setupCluster() throws IOException {
conf = new HdfsConfiguration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
cluster.waitActive();
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index 8f83ba5..1c75f90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -22,9 +22,11 @@ import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -44,16 +46,17 @@ import java.io.IOException;
public class TestFileChecksum {
private static final Logger LOG = LoggerFactory
.getLogger(TestFileChecksum.class);
-
- private int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private int dataBlocks = ecPolicy.getNumDataUnits();
+ private int parityBlocks = ecPolicy.getNumParityUnits();
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private Configuration conf;
private DFSClient client;
- private int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+ private int cellSize = ecPolicy.getCellSize();
private int stripesPerBlock = 6;
private int blockSize = cellSize * stripesPerBlock;
private int numBlockGroups = 10;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index f7bac28..ca8f6db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -55,22 +56,19 @@ public class TestLeaseRecoveryStriped {
public static final Log LOG = LogFactory
.getLog(TestLeaseRecoveryStriped.class);
- private static final ErasureCodingPolicy ecPolicy =
- StripedFileTestUtil.TEST_EC_POLICY;
- private static final int NUM_DATA_BLOCKS = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private static final int NUM_PARITY_BLOCKS = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private static final int CELL_SIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private static final int STRIPE_SIZE = NUM_DATA_BLOCKS * CELL_SIZE;
- private static final int STRIPES_PER_BLOCK = 15;
- private static final int BLOCK_SIZE = CELL_SIZE * STRIPES_PER_BLOCK;
- private static final int BLOCK_GROUP_SIZE = BLOCK_SIZE * NUM_DATA_BLOCKS;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripSize = dataBlocks * cellSize;
+ private final int stripesPerBlock = 15;
+ private final int blockSize = cellSize * stripesPerBlock;
+ private final int blockGroupSize = blockSize * dataBlocks;
private static final int bytesPerChecksum = 512;
static {
GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
- StripedFileTestUtil.stripesPerBlock = STRIPES_PER_BLOCK;
- StripedFileTestUtil.blockSize = BLOCK_SIZE;
- StripedFileTestUtil.BLOCK_GROUP_SIZE = BLOCK_GROUP_SIZE;
}
static private final String fakeUsername = "fakeUser1";
@@ -85,13 +83,13 @@ public class TestLeaseRecoveryStriped {
@Before
public void setup() throws IOException {
conf = new HdfsConfiguration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 6000L);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
- final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ final int numDNs = dataBlocks + parityBlocks;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
@@ -106,7 +104,7 @@ public class TestLeaseRecoveryStriped {
}
}
- private static int[][][] getBlockLengthsSuite() {
+ private int[][][] getBlockLengthsSuite() {
final int groups = 4;
final int minNumCell = 3;
final int maxNumCell = 11;
@@ -120,13 +118,13 @@ public class TestLeaseRecoveryStriped {
delta = bytesPerChecksum;
}
int[][] suite = new int[2][];
- int[] lens = new int[NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS];
+ int[] lens = new int[dataBlocks + parityBlocks];
long[] lenInLong = new long[lens.length];
for (int j = 0; j < lens.length; j++) {
int numCell = random.nextInt(maxNumCell - minNumCell + 1) + minNumCell;
- int numDelta = j < NUM_DATA_BLOCKS ?
+ int numDelta = j < dataBlocks ?
random.nextInt(maxNumDelta - minNumDelta + 1) + minNumDelta : 0;
- lens[j] = CELL_SIZE * numCell + delta * numDelta;
+ lens[j] = cellSize * numCell + delta * numDelta;
lenInLong[j] = lens[j];
}
suite[0] = lens;
@@ -137,13 +135,13 @@ public class TestLeaseRecoveryStriped {
return blkLenSuite;
}
- private static final int[][][] BLOCK_LENGTHS_SUITE = getBlockLengthsSuite();
+ private final int[][][] blockLengthsSuite = getBlockLengthsSuite();
@Test
public void testLeaseRecovery() throws Exception {
- for (int i = 0; i < BLOCK_LENGTHS_SUITE.length; i++) {
- int[] blockLengths = BLOCK_LENGTHS_SUITE[i][0];
- int safeLength = BLOCK_LENGTHS_SUITE[i][1][0];
+ for (int i = 0; i < blockLengthsSuite.length; i++) {
+ int[] blockLengths = blockLengthsSuite[i][0];
+ int safeLength = blockLengthsSuite[i][1][0];
try {
runTest(blockLengths, safeLength);
} catch (Throwable e) {
@@ -162,20 +160,20 @@ public class TestLeaseRecoveryStriped {
List<Long> oldGS = new ArrayList<>();
oldGS.add(1001L);
StripedFileTestUtil.checkData(dfs, p, safeLength,
- new ArrayList<DatanodeInfo>(), oldGS);
+ new ArrayList<DatanodeInfo>(), oldGS, blockGroupSize);
// After recovery, storages are reported by primary DN. we should verify
// storages reported by blockReport.
cluster.restartNameNode(true);
cluster.waitFirstBRCompleted(0, 10000);
StripedFileTestUtil.checkData(dfs, p, safeLength,
- new ArrayList<DatanodeInfo>(), oldGS);
+ new ArrayList<DatanodeInfo>(), oldGS, blockGroupSize);
}
private void writePartialBlocks(int[] blockLengths) throws Exception {
final FSDataOutputStream out = dfs.create(p);
final DFSStripedOutputStream stripedOut = (DFSStripedOutputStream) out
.getWrappedStream();
- int length = (STRIPES_PER_BLOCK - 1) * STRIPE_SIZE;
+ int length = (stripesPerBlock - 1) * stripSize;
int[] posToKill = getPosToKill(blockLengths);
int checkingPos = nextCheckingPos(posToKill, 0);
try {
@@ -209,20 +207,20 @@ public class TestLeaseRecoveryStriped {
}
private int[] getPosToKill(int[] blockLengths) {
- int[] posToKill = new int[NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS];
- for (int i = 0; i < NUM_DATA_BLOCKS; i++) {
- int numStripe = (blockLengths[i] - 1) / CELL_SIZE;
- posToKill[i] = numStripe * STRIPE_SIZE + i * CELL_SIZE
- + blockLengths[i] % CELL_SIZE;
- if (blockLengths[i] % CELL_SIZE == 0) {
- posToKill[i] += CELL_SIZE;
+ int[] posToKill = new int[dataBlocks + parityBlocks];
+ for (int i = 0; i < dataBlocks; i++) {
+ int numStripe = (blockLengths[i] - 1) / cellSize;
+ posToKill[i] = numStripe * stripSize + i * cellSize
+ + blockLengths[i] % cellSize;
+ if (blockLengths[i] % cellSize == 0) {
+ posToKill[i] += cellSize;
}
}
- for (int i = NUM_DATA_BLOCKS; i < NUM_DATA_BLOCKS
- + NUM_PARITY_BLOCKS; i++) {
- Preconditions.checkArgument(blockLengths[i] % CELL_SIZE == 0);
- int numStripe = (blockLengths[i]) / CELL_SIZE;
- posToKill[i] = numStripe * STRIPE_SIZE;
+ for (int i = dataBlocks; i < dataBlocks
+ + parityBlocks; i++) {
+ Preconditions.checkArgument(blockLengths[i] % cellSize == 0);
+ int numStripe = (blockLengths[i]) / cellSize;
+ posToKill[i] = numStripe * stripSize;
}
return posToKill;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 6d2227f..6014332 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -52,9 +54,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
-
public class TestReadStripedFileWithDecoding {
static final Log LOG = LogFactory.getLog(TestReadStripedFileWithDecoding.class);
@@ -68,15 +67,22 @@ public class TestReadStripedFileWithDecoding {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
- private static final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private static final short parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private final int smallFileLength = blockSize * dataBlocks - 123;
- private final int largeFileLength = blockSize * dataBlocks + 123;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks =
+ (short) ecPolicy.getNumParityUnits();
+ private final int numDNs = dataBlocks + parityBlocks;
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripPerBlock = 4;
+ private final int blockSize = cellSize * stripPerBlock;
+ private final int blockGroupSize = blockSize * dataBlocks;
+ private final int smallFileLength = blockGroupSize - 123;
+ private final int largeFileLength = blockGroupSize + 123;
private final int[] fileLengths = {smallFileLength, largeFileLength};
- private static final int[] dnFailureNums = getDnFailureNums();
+ private final int[] dnFailureNums = getDnFailureNums();
- private static int[] getDnFailureNums() {
+ private int[] getDnFailureNums() {
int[] dnFailureNums = new int[parityBlocks];
for (int i = 0; i < dnFailureNums.length; i++) {
dnFailureNums[i] = i + 1;
@@ -191,7 +197,8 @@ public class TestReadStripedFileWithDecoding {
StripedFileTestUtil.verifyStatefulRead(fs, testPath, length, expected, buffer);
StripedFileTestUtil.verifyStatefulRead(fs, testPath, length, expected,
ByteBuffer.allocate(length + 100));
- StripedFileTestUtil.verifySeek(fs, testPath, length);
+ StripedFileTestUtil.verifySeek(fs, testPath, length, ecPolicy,
+ blockGroupSize);
}
private void testReadWithDNFailure(int fileLength, int dnFailureNum)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
index fe89401..2ee8663 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -22,7 +22,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.Rule;
@@ -30,11 +32,6 @@ import org.junit.rules.Timeout;
import java.io.IOException;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.TEST_EC_POLICY;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
-
/**
* Test reading a striped file when some of its blocks are missing (not included
* in the block locations returned by the NameNode).
@@ -45,8 +42,15 @@ public class TestReadStripedFileWithMissingBlocks {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private Configuration conf = new HdfsConfiguration();
- private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripPerBlock = 4;
+ private final int blockSize = stripPerBlock * cellSize;
+ private final int blockGroupSize = blockSize * dataBlocks;
+ private final int numDNs = dataBlocks + parityBlocks;
private final int fileLength = blockSize * dataBlocks + 123;
@Rule
@@ -57,7 +61,7 @@ public class TestReadStripedFileWithMissingBlocks {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy(
- "/", TEST_EC_POLICY);
+ "/", ecPolicy);
fs = cluster.getFileSystem();
}
@@ -70,9 +74,9 @@ public class TestReadStripedFileWithMissingBlocks {
@Test
public void testReadFileWithMissingBlocks() throws Exception {
- for (int missingData = 1; missingData <= NUM_PARITY_BLOCKS; missingData++) {
+ for (int missingData = 1; missingData <= dataBlocks; missingData++) {
for (int missingParity = 0; missingParity <=
- NUM_PARITY_BLOCKS - missingData; missingParity++) {
+ parityBlocks - missingData; missingParity++) {
try {
setup();
readFileWithMissingBlocks(new Path("/foo"), fileLength,
@@ -102,7 +106,7 @@ public class TestReadStripedFileWithMissingBlocks {
}
for (int i = 0; i < missingParityNum; i++) {
missingDataNodes[i + missingDataNum] = i +
- Math.min(StripedFileTestUtil.NUM_DATA_BLOCKS, dataBlocks);
+ Math.min(ecPolicy.getNumDataUnits(), dataBlocks);
}
stopDataNodes(locs, missingDataNodes);
@@ -112,7 +116,8 @@ public class TestReadStripedFileWithMissingBlocks {
byte[] smallBuf = new byte[1024];
byte[] largeBuf = new byte[fileLength + 100];
- StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy,
+ blockGroupSize);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
smallBuf);
StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 10fc43e..a3d9ef6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -60,12 +61,14 @@ import org.junit.Test;
public class TestReconstructStripedFile {
public static final Log LOG = LogFactory.getLog(TestReconstructStripedFile.class);
- private static final int dataBlkNum = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private static final int parityBlkNum = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private static final int blockSize = cellSize * 3;
- private static final int groupSize = dataBlkNum + parityBlkNum;
- private static final int dnNum = groupSize + parityBlkNum;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlkNum = ecPolicy.getNumDataUnits();
+ private final int parityBlkNum = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int blockSize = cellSize * 3;
+ private final int groupSize = dataBlkNum + parityBlkNum;
+ private final int dnNum = groupSize + parityBlkNum;
static {
GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
index 2b6b65a..2d37c06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.After;
@@ -41,11 +43,13 @@ import static org.junit.Assert.assertTrue;
public class TestSafeModeWithStripedFile {
- static final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS;
- static final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- static final int numDNs = DATA_BLK_NUM + PARITY_BLK_NUM;
- static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- static final int blockSize = cellSize * 2;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int numDNs = dataBlocks + parityBlocks;
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int blockSize = cellSize * 2;
private MiniDFSCluster cluster;
private Configuration conf;
@@ -78,7 +82,7 @@ public class TestSafeModeWithStripedFile {
@Test
public void testStripedFile1() throws IOException {
- int numCell = DATA_BLK_NUM - 1;
+ int numCell = dataBlocks - 1;
doTest(cellSize * numCell, numCell);
}
@@ -101,7 +105,7 @@ public class TestSafeModeWithStripedFile {
// If we only have 1 block, NN won't enter safemode in the first place
// because the threshold is 0 blocks.
// So we need to add another 2 blocks.
- int bigSize = blockSize * DATA_BLK_NUM * 2;
+ int bigSize = blockSize * dataBlocks * 2;
Path bigFilePath = new Path("/testStripedFile_" + bigSize);
data = StripedFileTestUtil.generateBytes(bigSize);
DFSTestUtil.writeFile(fs, bigFilePath, data);
@@ -143,7 +147,7 @@ public class TestSafeModeWithStripedFile {
assertEquals(1, NameNodeAdapter.getSafeModeSafeBlocks(nn));
// the 2 blocks of bigFile need DATA_BLK_NUM storages to be safe
- for (int i = minStorages; i < DATA_BLK_NUM - 1; i++) {
+ for (int i = minStorages; i < dataBlocks - 1; i++) {
cluster.restartDataNode(dnprops.remove(0));
cluster.waitActive();
cluster.triggerBlockReports();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 0836656..8ada593 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -24,8 +24,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
import org.apache.hadoop.ipc.RemoteException;
@@ -43,14 +45,18 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Random;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.stripesPerBlock;
-
public class TestWriteReadStripedFile {
public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class);
- private static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private static short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int numDNs = dataBlocks + parityBlocks;
+ private final int stripesPerBlock = 4;
+ private final int blockSize = stripesPerBlock * cellSize;
+ private final int blockGroupSize = blockSize * dataBlocks;
+
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private Configuration conf = new HdfsConfiguration();
@@ -221,7 +227,8 @@ public class TestWriteReadStripedFile {
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
largeBuf);
- StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy,
+ blockGroupSize);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
ByteBuffer.allocate(fileLength + 100));
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
@@ -263,7 +270,8 @@ public class TestWriteReadStripedFile {
//StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, largeBuf);
- StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy,
+ blockGroupSize);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf);
// webhdfs doesn't support bytebuffer read
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
index fad2dcd..b89e169 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Assert;
@@ -32,9 +34,6 @@ import org.junit.Test;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
-
public class TestWriteStripedFileWithFailure {
public static final Log LOG = LogFactory
.getLog(TestWriteStripedFileWithFailure.class);
@@ -47,8 +46,12 @@ public class TestWriteStripedFileWithFailure {
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
}
- private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private final short parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int numDNs = dataBlocks + parityBlocks;
+ private final int blockSize = 4 * ecPolicy.getCellSize();
private final int smallFileLength = blockSize * dataBlocks - 123;
private final int largeFileLength = blockSize * dataBlocks + 123;
private final int[] fileLengths = {smallFileLength, largeFileLength};
@@ -153,7 +156,8 @@ public class TestWriteStripedFileWithFailure {
byte[] largeBuf = new byte[fileLength + 100];
final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
- StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy,
+ blockSize * dataBlocks);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
smallBuf);
StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 5cb2571..eedf08d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -212,7 +211,7 @@ public class TestPBHelper {
datanodeUuids, storageIDs, storageTypes);
if (isStriped) {
blkLocs = new StripedBlockWithLocations(blkLocs, indices, dataBlkNum,
- StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE);
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getCellSize());
}
return blkLocs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 22b48ad..b63aa4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -44,6 +44,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.junit.AfterClass;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -196,15 +199,17 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
}
- int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- int groupSize = dataBlocks + parityBlocks;
- private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private final static int stripesPerBlock = 4;
- static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int groupSize = dataBlocks + parityBlocks;
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripesPerBlock = 4;
+ private final int defaultBlockSize = cellSize * stripesPerBlock;
- static void initConfWithStripe(Configuration conf) {
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_STRIPE_BLOCK_SIZE);
+ void initConfWithStripe(Configuration conf) {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
SimulatedFSDataset.setFactory(conf);
@@ -1908,7 +1913,7 @@ public class TestBalancer {
private void doTestBalancerWithStripedFile(Configuration conf) throws Exception {
int numOfDatanodes = dataBlocks + parityBlocks + 2;
int numOfRacks = dataBlocks;
- long capacity = 20 * DEFAULT_STRIPE_BLOCK_SIZE;
+ long capacity = 20 * defaultBlockSize;
long[] capacities = new long[numOfDatanodes];
for (int i = 0; i < capacities.length; i++) {
capacities[i] = capacity;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
index 2fc454b..255127f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.junit.Assert;
@@ -33,8 +32,6 @@ import java.io.DataOutputStream;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -43,11 +40,12 @@ import static org.junit.Assert.fail;
* Test {@link BlockInfoStriped}
*/
public class TestBlockInfoStriped {
- private static final int TOTAL_NUM_BLOCKS = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
private static final long BASE_ID = -1600;
- private static final Block baseBlock = new Block(BASE_ID);
- private static final ErasureCodingPolicy testECPolicy
+ private final Block baseBlock = new Block(BASE_ID);
+ private final ErasureCodingPolicy testECPolicy
= ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int totalBlocks = testECPolicy.getNumDataUnits() +
+ testECPolicy.getNumParityUnits();
private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
testECPolicy);
@@ -70,8 +68,8 @@ public class TestBlockInfoStriped {
// first add NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS storages, i.e., a complete
// group of blocks/storages
DatanodeStorageInfo[] storageInfos = DFSTestUtil.createDatanodeStorageInfos(
- TOTAL_NUM_BLOCKS);
- Block[] blocks = createReportedBlocks(TOTAL_NUM_BLOCKS);
+ totalBlocks);
+ Block[] blocks = createReportedBlocks(totalBlocks);
int i = 0;
for (; i < storageInfos.length; i += 2) {
info.addStorage(storageInfos[i], blocks[i]);
@@ -85,8 +83,8 @@ public class TestBlockInfoStriped {
// check
byte[] indices = (byte[]) Whitebox.getInternalState(info, "indices");
- Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
- Assert.assertEquals(TOTAL_NUM_BLOCKS, indices.length);
+ Assert.assertEquals(totalBlocks, info.getCapacity());
+ Assert.assertEquals(totalBlocks, indices.length);
i = 0;
for (DatanodeStorageInfo storage : storageInfos) {
int index = info.findStorageInfo(storage);
@@ -99,9 +97,9 @@ public class TestBlockInfoStriped {
for (DatanodeStorageInfo storage : storageInfos) {
Assert.assertTrue(info.addStorage(storage, blocks[i++]));
}
- Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
- Assert.assertEquals(TOTAL_NUM_BLOCKS, info.numNodes());
- Assert.assertEquals(TOTAL_NUM_BLOCKS, indices.length);
+ Assert.assertEquals(totalBlocks, info.getCapacity());
+ Assert.assertEquals(totalBlocks, info.numNodes());
+ Assert.assertEquals(totalBlocks, indices.length);
i = 0;
for (DatanodeStorageInfo storage : storageInfos) {
int index = info.findStorageInfo(storage);
@@ -111,19 +109,19 @@ public class TestBlockInfoStriped {
// the same block is reported from another storage
DatanodeStorageInfo[] storageInfos2 = DFSTestUtil.createDatanodeStorageInfos(
- TOTAL_NUM_BLOCKS * 2);
+ totalBlocks * 2);
// only add the second half of info2
- for (i = TOTAL_NUM_BLOCKS; i < storageInfos2.length; i++) {
- info.addStorage(storageInfos2[i], blocks[i % TOTAL_NUM_BLOCKS]);
+ for (i = totalBlocks; i < storageInfos2.length; i++) {
+ info.addStorage(storageInfos2[i], blocks[i % totalBlocks]);
Assert.assertEquals(i + 1, info.getCapacity());
Assert.assertEquals(i + 1, info.numNodes());
indices = (byte[]) Whitebox.getInternalState(info, "indices");
Assert.assertEquals(i + 1, indices.length);
}
- for (i = TOTAL_NUM_BLOCKS; i < storageInfos2.length; i++) {
+ for (i = totalBlocks; i < storageInfos2.length; i++) {
int index = info.findStorageInfo(storageInfos2[i]);
Assert.assertEquals(i++, index);
- Assert.assertEquals(index - TOTAL_NUM_BLOCKS, indices[index]);
+ Assert.assertEquals(index - totalBlocks, indices[index]);
}
}
@@ -131,8 +129,8 @@ public class TestBlockInfoStriped {
public void testRemoveStorage() {
// first add TOTAL_NUM_BLOCKS into the BlockInfoStriped
DatanodeStorageInfo[] storages = DFSTestUtil.createDatanodeStorageInfos(
- TOTAL_NUM_BLOCKS);
- Block[] blocks = createReportedBlocks(TOTAL_NUM_BLOCKS);
+ totalBlocks);
+ Block[] blocks = createReportedBlocks(totalBlocks);
for (int i = 0; i < storages.length; i++) {
info.addStorage(storages[i], blocks[i]);
}
@@ -142,8 +140,8 @@ public class TestBlockInfoStriped {
info.removeStorage(storages[2]);
// check
- Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
- Assert.assertEquals(TOTAL_NUM_BLOCKS - 2, info.numNodes());
+ Assert.assertEquals(totalBlocks, info.getCapacity());
+ Assert.assertEquals(totalBlocks - 2, info.numNodes());
byte[] indices = (byte[]) Whitebox.getInternalState(info, "indices");
for (int i = 0; i < storages.length; i++) {
int index = info.findStorageInfo(storages[i]);
@@ -158,44 +156,44 @@ public class TestBlockInfoStriped {
// the same block is reported from another storage
DatanodeStorageInfo[] storages2 = DFSTestUtil.createDatanodeStorageInfos(
- TOTAL_NUM_BLOCKS * 2);
- for (int i = TOTAL_NUM_BLOCKS; i < storages2.length; i++) {
- info.addStorage(storages2[i], blocks[i % TOTAL_NUM_BLOCKS]);
+ totalBlocks * 2);
+ for (int i = totalBlocks; i < storages2.length; i++) {
+ info.addStorage(storages2[i], blocks[i % totalBlocks]);
}
// now we should have 8 storages
- Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.numNodes());
- Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.getCapacity());
+ Assert.assertEquals(totalBlocks * 2 - 2, info.numNodes());
+ Assert.assertEquals(totalBlocks * 2 - 2, info.getCapacity());
indices = (byte[]) Whitebox.getInternalState(info, "indices");
- Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, indices.length);
- int j = TOTAL_NUM_BLOCKS;
- for (int i = TOTAL_NUM_BLOCKS; i < storages2.length; i++) {
+ Assert.assertEquals(totalBlocks * 2 - 2, indices.length);
+ int j = totalBlocks;
+ for (int i = totalBlocks; i < storages2.length; i++) {
int index = info.findStorageInfo(storages2[i]);
- if (i == TOTAL_NUM_BLOCKS || i == TOTAL_NUM_BLOCKS + 2) {
- Assert.assertEquals(i - TOTAL_NUM_BLOCKS, index);
+ if (i == totalBlocks || i == totalBlocks + 2) {
+ Assert.assertEquals(i - totalBlocks, index);
} else {
Assert.assertEquals(j++, index);
}
}
// remove the storages from storages2
- for (int i = 0; i < TOTAL_NUM_BLOCKS; i++) {
- info.removeStorage(storages2[i + TOTAL_NUM_BLOCKS]);
+ for (int i = 0; i < totalBlocks; i++) {
+ info.removeStorage(storages2[i + totalBlocks]);
}
// now we should have 3 storages
- Assert.assertEquals(TOTAL_NUM_BLOCKS - 2, info.numNodes());
- Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.getCapacity());
+ Assert.assertEquals(totalBlocks - 2, info.numNodes());
+ Assert.assertEquals(totalBlocks * 2 - 2, info.getCapacity());
indices = (byte[]) Whitebox.getInternalState(info, "indices");
- Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, indices.length);
- for (int i = 0; i < TOTAL_NUM_BLOCKS; i++) {
+ Assert.assertEquals(totalBlocks * 2 - 2, indices.length);
+ for (int i = 0; i < totalBlocks; i++) {
if (i == 0 || i == 2) {
- int index = info.findStorageInfo(storages2[i + TOTAL_NUM_BLOCKS]);
+ int index = info.findStorageInfo(storages2[i + totalBlocks]);
Assert.assertEquals(-1, index);
} else {
int index = info.findStorageInfo(storages[i]);
Assert.assertEquals(i, index);
}
}
- for (int i = TOTAL_NUM_BLOCKS; i < TOTAL_NUM_BLOCKS * 2 - 2; i++) {
+ for (int i = totalBlocks; i < totalBlocks * 2 - 2; i++) {
Assert.assertEquals(-1, indices[i]);
Assert.assertNull(info.getDatanode(i));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 1714561..834df6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -20,10 +20,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.ServerSocketUtil;
import org.junit.Rule;
@@ -33,12 +34,13 @@ import org.junit.rules.Timeout;
import java.io.IOException;
public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
-
- private final static int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private final static int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private final static int stripesPerBlock = 4;
- private final static int numDNs = dataBlocks + parityBlocks + 2;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripesPerBlock = 4;
+ private final int numDNs = dataBlocks + parityBlocks + 2;
private MiniDFSCluster cluster;
private Configuration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
index 3bc13a8..94ee623 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -24,10 +24,12 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.net.NetworkTopology;
@@ -46,10 +48,6 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
-
public class TestReconstructStripedBlocksWithRackAwareness {
public static final Logger LOG = LoggerFactory.getLogger(
TestReconstructStripedBlocksWithRackAwareness.class);
@@ -60,10 +58,14 @@ public class TestReconstructStripedBlocksWithRackAwareness {
GenericTestUtils.setLogLevel(BlockManager.LOG, Level.ALL);
}
- private static final String[] hosts =
- getHosts(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1);
- private static final String[] racks =
- getRacks(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1, NUM_DATA_BLOCKS);
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final String[] hosts = getHosts(dataBlocks + parityBlocks + 1);
+ private final String[] racks =
+ getRacks(dataBlocks + parityBlocks + 1, dataBlocks);
private static String[] getHosts(int numHosts) {
String[] hosts = new String[numHosts];
@@ -157,7 +159,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
final Path file = new Path("/foo");
// the file's block is in 9 dn but 5 racks
DFSTestUtil.createFile(fs, file,
- BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+ cellSize * dataBlocks * 2, (short) 1, 0L);
Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
final INodeFile fileNode = fsn.getFSDirectory()
@@ -169,7 +171,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
for (DatanodeStorageInfo storage : blockInfo.storages) {
rackSet.add(storage.getDatanodeDescriptor().getNetworkLocation());
}
- Assert.assertEquals(NUM_DATA_BLOCKS - 1, rackSet.size());
+ Assert.assertEquals(dataBlocks - 1, rackSet.size());
// restart the stopped datanode
cluster.restartDataNode(lastHost);
@@ -178,7 +180,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
// make sure we have 6 racks again
NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
Assert.assertEquals(hosts.length, topology.getNumOfLeaves());
- Assert.assertEquals(NUM_DATA_BLOCKS, topology.getNumOfRacks());
+ Assert.assertEquals(dataBlocks, topology.getNumOfRacks());
// pause all the heartbeats
for (DataNode dn : cluster.getDataNodes()) {
@@ -225,7 +227,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
final Path file = new Path("/foo");
DFSTestUtil.createFile(fs, file,
- BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+ cellSize * dataBlocks * 2, (short) 1, 0L);
// stop host1
MiniDFSCluster.DataNodeProperties host1 = stopDataNode("host1");
@@ -234,7 +236,7 @@ public class TestReconstructStripedBlocksWithRackAwareness {
cluster.waitActive();
// wait for reconstruction to finish
- final short blockNum = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+ final short blockNum = (short) (dataBlocks + parityBlocks);
DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
// restart host1
@@ -263,12 +265,12 @@ public class TestReconstructStripedBlocksWithRackAwareness {
*/
@Test
public void testReconstructionWithDecommission() throws Exception {
- final String[] racks = getRacks(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2,
- NUM_DATA_BLOCKS);
- final String[] hosts = getHosts(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2);
+ final String[] rackNames = getRacks(dataBlocks + parityBlocks + 2,
+ dataBlocks);
+ final String[] hostNames = getHosts(dataBlocks + parityBlocks + 2);
// we now have 11 hosts on 6 racks with distribution: 2-2-2-2-2-1
- cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
- .numDataNodes(hosts.length).build();
+ cluster = new MiniDFSCluster.Builder(conf).racks(rackNames).hosts(hostNames)
+ .numDataNodes(hostNames.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
fs.setErasureCodingPolicy(new Path("/"), null);
@@ -277,11 +279,13 @@ public class TestReconstructStripedBlocksWithRackAwareness {
final DatanodeManager dm = bm.getDatanodeManager();
// stop h9 and h10 and create a file with 6+3 internal blocks
- MiniDFSCluster.DataNodeProperties h9 = stopDataNode(hosts[hosts.length - 3]);
- MiniDFSCluster.DataNodeProperties h10 = stopDataNode(hosts[hosts.length - 2]);
+ MiniDFSCluster.DataNodeProperties h9 =
+ stopDataNode(hostNames[hostNames.length - 3]);
+ MiniDFSCluster.DataNodeProperties h10 =
+ stopDataNode(hostNames[hostNames.length - 2]);
final Path file = new Path("/foo");
DFSTestUtil.createFile(fs, file,
- BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+ cellSize * dataBlocks * 2, (short) 1, 0L);
final BlockInfo blockInfo = cluster.getNamesystem().getFSDirectory()
.getINode(file.toString()).asFile().getLastBlock();
@@ -290,18 +294,19 @@ public class TestReconstructStripedBlocksWithRackAwareness {
cluster.waitActive();
// stop h11 so that the reconstruction happens
- MiniDFSCluster.DataNodeProperties h11 = stopDataNode(hosts[hosts.length - 1]);
+ MiniDFSCluster.DataNodeProperties h11 =
+ stopDataNode(hostNames[hostNames.length - 1]);
boolean recovered = bm.countNodes(blockInfo).liveReplicas() >=
- NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ dataBlocks + parityBlocks;
for (int i = 0; i < 10 & !recovered; i++) {
Thread.sleep(1000);
recovered = bm.countNodes(blockInfo).liveReplicas() >=
- NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ dataBlocks + parityBlocks;
}
Assert.assertTrue(recovered);
// mark h9 as decommissioning
- DataNode datanode9 = getDataNode(hosts[hosts.length - 3]);
+ DataNode datanode9 = getDataNode(hostNames[hostNames.length - 3]);
Assert.assertNotNull(datanode9);
final DatanodeDescriptor dn9 = dm.getDatanode(datanode9.getDatanodeId());
dn9.startDecommission();
@@ -310,7 +315,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
cluster.restartDataNode(h10);
cluster.restartDataNode(h11);
cluster.waitActive();
- DataNodeTestUtils.triggerBlockReport(getDataNode(hosts[hosts.length - 1]));
+ DataNodeTestUtils.triggerBlockReport(
+ getDataNode(hostNames[hostNames.length - 1]));
// start decommissioning h9
boolean satisfied = bm.isPlacementPolicySatisfied(blockInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
index 41a7878..10ea01f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
@@ -37,8 +37,9 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
@@ -56,11 +57,13 @@ public class TestSequentialBlockGroupId {
private static final Log LOG = LogFactory
.getLog("TestSequentialBlockGroupId");
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
private final short REPLICATION = 1;
private final long SEED = 0;
- private final int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private final int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
private final int stripesPerBlock = 2;
private final int blockSize = cellSize * stripesPerBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
index 1dd067d..4db3617 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
@@ -27,13 +27,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
@@ -51,10 +52,14 @@ import org.slf4j.LoggerFactory;
public class TestSortLocatedStripedBlock {
static final Logger LOG = LoggerFactory
.getLogger(TestSortLocatedStripedBlock.class);
- static final int BLK_GROUP_WIDTH = StripedFileTestUtil.NUM_DATA_BLOCKS
- + StripedFileTestUtil.NUM_PARITY_BLOCKS;
- static final int NUM_DATA_BLOCKS = StripedFileTestUtil.NUM_DATA_BLOCKS;
- static final int NUM_PARITY_BLOCKS = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+ private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+ private final int groupSize = dataBlocks + parityBlocks;
+
static DatanodeManager dm;
static final long STALE_INTERVAL = 30 * 1000 * 60;
@@ -100,7 +105,7 @@ public class TestSortLocatedStripedBlock {
HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
lbsCount * decommnNodeIndices.size());
List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
- NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ dataBlocks, parityBlocks, decommnNodeIndices,
targetNodeIndices, decommissionedNodes);
// prepare expected block index and token list.
@@ -111,7 +116,7 @@ public class TestSortLocatedStripedBlock {
dm.sortLocatedBlocks(null, lbs);
- assertDecommnNodePosition(BLK_GROUP_WIDTH, decommissionedNodes, lbs);
+ assertDecommnNodePosition(groupSize, decommissionedNodes, lbs);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
}
@@ -156,7 +161,7 @@ public class TestSortLocatedStripedBlock {
HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
lbsCount * decommnNodeIndices.size());
List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
- NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ dataBlocks, parityBlocks, decommnNodeIndices,
targetNodeIndices, decommissionedNodes);
// prepare expected block index and token list.
@@ -166,7 +171,7 @@ public class TestSortLocatedStripedBlock {
prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
dm.sortLocatedBlocks(null, lbs);
- assertDecommnNodePosition(BLK_GROUP_WIDTH, decommissionedNodes, lbs);
+ assertDecommnNodePosition(groupSize, decommissionedNodes, lbs);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
}
@@ -209,9 +214,9 @@ public class TestSortLocatedStripedBlock {
// which will be used for assertions
HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
lbsCount * decommnNodeIndices.size());
- int dataBlksNum = NUM_DATA_BLOCKS - 2;
+ int dataBlksNum = dataBlocks - 2;
List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount, dataBlksNum,
- NUM_PARITY_BLOCKS, decommnNodeIndices, targetNodeIndices,
+ parityBlocks, decommnNodeIndices, targetNodeIndices,
decommissionedNodes);
// prepare expected block index and token list.
@@ -223,7 +228,7 @@ public class TestSortLocatedStripedBlock {
dm.sortLocatedBlocks(null, lbs);
// After this index all are decommissioned nodes.
- int blkGrpWidth = dataBlksNum + NUM_PARITY_BLOCKS;
+ int blkGrpWidth = dataBlksNum + parityBlocks;
assertDecommnNodePosition(blkGrpWidth, decommissionedNodes, lbs);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
}
@@ -275,7 +280,7 @@ public class TestSortLocatedStripedBlock {
HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
lbsCount * decommnNodeIndices.size());
List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
- NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ dataBlocks, parityBlocks, decommnNodeIndices,
targetNodeIndices, decommissionedNodes);
// prepare expected block index and token list.
@@ -288,7 +293,7 @@ public class TestSortLocatedStripedBlock {
// After this index all are decommissioned nodes. Needs to reconstruct two
// more block indices.
- int blkGrpWidth = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS - 2;
+ int blkGrpWidth = dataBlocks + parityBlocks - 2;
assertDecommnNodePosition(blkGrpWidth, decommissionedNodes, lbs);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
}
@@ -336,7 +341,7 @@ public class TestSortLocatedStripedBlock {
HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
lbsCount * decommnNodeIndices.size());
List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
- NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ dataBlocks, parityBlocks, decommnNodeIndices,
targetNodeIndices, decommissionedNodes);
List <DatanodeInfo> staleDns = new ArrayList<>();
for (LocatedBlock lb : lbs) {
@@ -355,7 +360,7 @@ public class TestSortLocatedStripedBlock {
dm.sortLocatedBlocks(null, lbs);
- assertDecommnNodePosition(BLK_GROUP_WIDTH + 1, decommissionedNodes, lbs);
+ assertDecommnNodePosition(groupSize + 1, decommissionedNodes, lbs);
assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
for (LocatedBlock lb : lbs) {
@@ -452,7 +457,7 @@ public class TestSortLocatedStripedBlock {
}
}
// Adding parity blocks after data blocks
- index = NUM_DATA_BLOCKS;
+ index = dataBlocks;
for (int j = numDataBlk; j < numDataBlk + numParityBlk; j++, index++) {
blkIndices[j] = (byte) index;
// Location port always equal to logical index of a block,
@@ -471,7 +476,7 @@ public class TestSortLocatedStripedBlock {
}
}
// Add extra target nodes to storage list after the parity blocks
- int basePortValue = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ int basePortValue = dataBlocks + parityBlocks;
index = numDataBlk + numParityBlk;
for (int i = 0; i < targetNodeIndices.size(); i++, index++) {
int blkIndexPos = targetNodeIndices.get(i);
@@ -494,7 +499,7 @@ public class TestSortLocatedStripedBlock {
}
return new LocatedStripedBlock(
new ExtendedBlock("pool", blockGroupID,
- StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE, 1001),
+ cellSize, 1001),
locs, storageIDs, storageTypes, blkIndices, 0, false, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index f011b9d..3ef4067 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -109,7 +108,7 @@ import org.mockito.stubbing.Answer;
import com.google.common.base.Supplier;
/**
- * This tests if sync all replicas in block recovery works correctly
+ * This tests if sync all replicas in block recovery works correctly.
*/
public class TestBlockRecovery {
private static final Log LOG = LogFactory.getLog(TestBlockRecovery.class);
@@ -136,30 +135,30 @@ public class TestBlockRecovery {
@Rule
public TestName currentTestName = new TestName();
- private static final int CELL_SIZE =
- StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private static final int bytesPerChecksum = 512;
- private static final int[][][] BLOCK_LENGTHS_SUITE = {
- { { 11 * CELL_SIZE, 10 * CELL_SIZE, 9 * CELL_SIZE, 8 * CELL_SIZE,
- 7 * CELL_SIZE, 6 * CELL_SIZE, 5 * CELL_SIZE, 4 * CELL_SIZE,
- 3 * CELL_SIZE }, { 36 * CELL_SIZE } },
-
- { { 3 * CELL_SIZE, 4 * CELL_SIZE, 5 * CELL_SIZE, 6 * CELL_SIZE,
- 7 * CELL_SIZE, 8 * CELL_SIZE, 9 * CELL_SIZE, 10 * CELL_SIZE,
- 11 * CELL_SIZE }, { 36 * CELL_SIZE } },
-
- { { 11 * CELL_SIZE, 7 * CELL_SIZE, 6 * CELL_SIZE, 5 * CELL_SIZE,
- 4 * CELL_SIZE, 2 * CELL_SIZE, 9 * CELL_SIZE, 10 * CELL_SIZE,
- 11 * CELL_SIZE }, { 36 * CELL_SIZE } },
-
- { { 8 * CELL_SIZE + bytesPerChecksum,
- 7 * CELL_SIZE + bytesPerChecksum * 2,
- 6 * CELL_SIZE + bytesPerChecksum * 2,
- 5 * CELL_SIZE - bytesPerChecksum * 3,
- 4 * CELL_SIZE - bytesPerChecksum * 4,
- 3 * CELL_SIZE - bytesPerChecksum * 4, 9 * CELL_SIZE, 10 * CELL_SIZE,
- 11 * CELL_SIZE }, { 36 * CELL_SIZE } }, };
-
+ private final int cellSize =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy().getCellSize();
+ private final int bytesPerChecksum = 512;
+ private final int[][][] blockLengthsSuite = {
+ {{11 * cellSize, 10 * cellSize, 9 * cellSize, 8 * cellSize,
+ 7 * cellSize, 6 * cellSize, 5 * cellSize, 4 * cellSize,
+ 3 * cellSize}, {36 * cellSize}},
+
+ {{3 * cellSize, 4 * cellSize, 5 * cellSize, 6 * cellSize,
+ 7 * cellSize, 8 * cellSize, 9 * cellSize, 10 * cellSize,
+ 11 * cellSize}, {36 * cellSize}},
+
+ {{11 * cellSize, 7 * cellSize, 6 * cellSize, 5 * cellSize,
+ 4 * cellSize, 2 * cellSize, 9 * cellSize, 10 * cellSize,
+ 11 * cellSize}, {36 * cellSize}},
+
+ {{8 * cellSize + bytesPerChecksum,
+ 7 * cellSize + bytesPerChecksum * 2,
+ 6 * cellSize + bytesPerChecksum * 2,
+ 5 * cellSize - bytesPerChecksum * 3,
+ 4 * cellSize - bytesPerChecksum * 4,
+ 3 * cellSize - bytesPerChecksum * 4, 9 * cellSize, 10 * cellSize,
+ 11 * cellSize}, {36 * cellSize}}, };
+
static {
GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
GenericTestUtils.setLogLevel(LOG, Level.ALL);
@@ -807,9 +806,9 @@ public class TestBlockRecovery {
BlockRecoveryWorker.RecoveryTaskStriped recoveryTask =
recoveryWorker.new RecoveryTaskStriped(rBlockStriped);
- for (int i = 0; i < BLOCK_LENGTHS_SUITE.length; i++) {
- int[] blockLengths = BLOCK_LENGTHS_SUITE[i][0];
- int safeLength = BLOCK_LENGTHS_SUITE[i][1][0];
+ for (int i = 0; i < blockLengthsSuite.length; i++) {
+ int[] blockLengths = blockLengthsSuite[i][0];
+ int safeLength = blockLengthsSuite[i][1][0];
Map<Long, BlockRecord> syncList = new HashMap<>();
for (int id = 0; id < blockLengths.length; id++) {
ReplicaRecoveryInfo rInfo = new ReplicaRecoveryInfo(id,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
index 825aa5a..98efc13c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -29,11 +29,13 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
@@ -58,15 +60,14 @@ import java.util.Arrays;
public class TestDataNodeErasureCodingMetrics {
public static final Log LOG = LogFactory.
getLog(TestDataNodeErasureCodingMetrics.class);
-
- private static final int DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private static final int PARITY_BLK_NUM =
- StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private static final int CELLSIZE =
- StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private static final int BLOCKSIZE = CELLSIZE;
- private static final int GROUPSIZE = DATA_BLK_NUM + PARITY_BLK_NUM;
- private static final int DN_NUM = GROUPSIZE + 1;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int blockSize = cellSize;
+ private final int groupSize = dataBlocks + parityBlocks;
+ private final int numDNs = groupSize + 1;
private MiniDFSCluster cluster;
private Configuration conf;
@@ -76,9 +77,9 @@ public class TestDataNodeErasureCodingMetrics {
public void setup() throws IOException {
conf = new Configuration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DN_NUM).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
fs = cluster.getFileSystem();
@@ -130,7 +131,7 @@ public class TestDataNodeErasureCodingMetrics {
private DataNode doTest(String fileName) throws Exception {
Path file = new Path(fileName);
- long fileLen = DATA_BLK_NUM * BLOCKSIZE;
+ long fileLen = dataBlocks * blockSize;
final byte[] data = StripedFileTestUtil.generateBytes((int) fileLen);
DFSTestUtil.writeFile(fs, file, data);
StripedFileTestUtil.waitBlockGroupsReported(fs, fileName);
@@ -142,7 +143,7 @@ public class TestDataNodeErasureCodingMetrics {
(LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
DataNode workerDn = null;
DatanodeInfo[] locations = lastBlock.getLocations();
- assertEquals(locations.length, GROUPSIZE);
+ assertEquals(locations.length, groupSize);
// we have ONE extra datanode in addition to the GROUPSIZE datanodes, here
// is to find the extra datanode that the reconstruction task will run on,
@@ -178,7 +179,7 @@ public class TestDataNodeErasureCodingMetrics {
int workCount = getComputedDatanodeWork();
assertTrue("Wrongly computed block reconstruction work", workCount > 0);
cluster.triggerHeartbeats();
- StripedFileTestUtil.waitForReconstructionFinished(file, fs, GROUPSIZE);
+ StripedFileTestUtil.waitForReconstructionFinished(file, fs, groupSize);
return workerDn;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index b7b750b..4342dab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -72,6 +73,7 @@ import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.minikdc.MiniKdc;
@@ -469,14 +471,16 @@ public class TestMover {
}
}
- int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
- private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
- private final static int stripesPerBlock = 4;
- static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock;
+ private final ErasureCodingPolicy ecPolicy =
+ ErasureCodingPolicyManager.getSystemDefaultPolicy();
+ private final int dataBlocks = ecPolicy.getNumDataUnits();
+ private final int parityBlocks = ecPolicy.getNumParityUnits();
+ private final int cellSize = ecPolicy.getCellSize();
+ private final int stripesPerBlock = 4;
+ private final int defaultBlockSize = cellSize * stripesPerBlock;
- static void initConfWithStripe(Configuration conf) {
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_STRIPE_BLOCK_SIZE);
+ void initConfWithStripe(Configuration conf) {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
@@ -490,7 +494,7 @@ public class TestMover {
// start 10 datanodes
int numOfDatanodes =10;
int storagesPerDatanode=2;
- long capacity = 10 * DEFAULT_STRIPE_BLOCK_SIZE;
+ long capacity = 10 * defaultBlockSize;
long[][] capacities = new long[numOfDatanodes][storagesPerDatanode];
for (int i = 0; i < numOfDatanodes; i++) {
for(int j=0;j<storagesPerDatanode;j++){
@@ -529,7 +533,7 @@ public class TestMover {
// write file to barDir
final String fooFile = "/bar/foo";
- long fileLen = 20 * DEFAULT_STRIPE_BLOCK_SIZE ;
+ long fileLen = 20 * defaultBlockSize;
DFSTestUtil.createFile(cluster.getFileSystem(), new Path(fooFile),
fileLen,(short) 3, 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org