You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/03/15 17:16:15 UTC
[2/3] hadoop git commit: HDFS-12723.
TestReadStripedFileWithMissingBlocks#testReadFileWithMissingBlocks failing
consistently. Contributed by Ajay Kumar.
HDFS-12723. TestReadStripedFileWithMissingBlocks#testReadFileWithMissingBlocks failing consistently. Contributed by Ajay Kumar.
(cherry picked from commit 6de135169eaaba9a4707d2bef380793ef91478d7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f6d33e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f6d33e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f6d33e
Branch: refs/heads/branch-3.1
Commit: 87f6d33e490f46f987ab5143aa2b07f98bbc3733
Parents: f4088e5
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Mar 15 10:14:35 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Mar 15 10:15:20 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 16 ++++++
.../TestReadStripedFileWithMissingBlocks.java | 59 +++++++++++++-------
2 files changed, 56 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f6d33e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 45c02f0..a3930c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2278,6 +2278,22 @@ public class MiniDFSCluster implements AutoCloseable {
}
/*
+ * Restart a DataNode by name.
+ * @return true if DataNode restart is successful else returns false
+ */
+ public synchronized boolean restartDataNode(String dnName)
+ throws IOException {
+ for (int i = 0; i < dataNodes.size(); i++) {
+ DataNode dn = dataNodes.get(i).datanode;
+ if (dnName.equals(dn.getDatanodeId().getXferAddr())) {
+ return restartDataNode(i);
+ }
+ }
+ return false;
+ }
+
+
+ /*
* Shutdown a particular datanode
* @param i node index
* @return null if the node index is out of range, else the properties of the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f6d33e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
index f3b8dd8..756adbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -22,7 +22,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.junit.Assert;
import org.junit.Test;
@@ -40,6 +42,7 @@ public class TestReadStripedFileWithMissingBlocks {
.getLog(TestReadStripedFileWithMissingBlocks.class);
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
+ private DFSClient dfsClient;
private Configuration conf = new HdfsConfiguration();
private final ErasureCodingPolicy ecPolicy =
StripedFileTestUtil.getDefaultECPolicy();
@@ -49,7 +52,9 @@ public class TestReadStripedFileWithMissingBlocks {
private final int stripPerBlock = 4;
private final int blockSize = stripPerBlock * cellSize;
private final int blockGroupSize = blockSize * dataBlocks;
- private final int numDNs = dataBlocks + parityBlocks;
+ // Starting with two more datanodes, minimum 9 should be up for
+ // test to pass.
+ private final int numDNs = dataBlocks + parityBlocks + 2;
private final int fileLength = blockSize * dataBlocks + 123;
@Rule
@@ -63,6 +68,8 @@ public class TestReadStripedFileWithMissingBlocks {
"/", ecPolicy.getName());
fs = cluster.getFileSystem();
fs.enableErasureCodingPolicy(ecPolicy.getName());
+ dfsClient = new DFSClient(cluster.getNameNode(0).getNameNodeAddress(),
+ conf);
}
public void tearDown() throws IOException {
@@ -74,29 +81,34 @@ public class TestReadStripedFileWithMissingBlocks {
@Test
public void testReadFileWithMissingBlocks() throws Exception {
- for (int missingData = 1; missingData <= dataBlocks; missingData++) {
- for (int missingParity = 0; missingParity <=
- parityBlocks - missingData; missingParity++) {
- try {
- setup();
- readFileWithMissingBlocks(new Path("/foo"), fileLength,
- missingData, missingParity);
- } finally {
- tearDown();
+ try {
+ setup();
+ Path srcPath = new Path("/foo");
+ final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
+ DFSTestUtil.writeFile(fs, srcPath, new String(expected));
+ StripedFileTestUtil
+ .waitBlockGroupsReported(fs, srcPath.toUri().getPath());
+ StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+
+ for (int missingData = 1; missingData <= dataBlocks; missingData++) {
+ for (int missingParity = 0; missingParity <=
+ parityBlocks - missingData; missingParity++) {
+ readFileWithMissingBlocks(srcPath, fileLength, missingData,
+ missingParity, expected);
}
}
+ } finally {
+ tearDown();
}
}
+
private void readFileWithMissingBlocks(Path srcPath, int fileLength,
- int missingDataNum, int missingParityNum)
+ int missingDataNum, int missingParityNum, byte[] expected)
throws Exception {
LOG.info("readFileWithMissingBlocks: (" + missingDataNum + ","
+ missingParityNum + ")");
- final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
- DFSTestUtil.writeFile(fs, srcPath, new String(expected));
- StripedFileTestUtil.waitBlockGroupsReported(fs, srcPath.toUri().getPath());
- StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+
int dataBlocks = (fileLength - 1) / cellSize + 1;
BlockLocation[] locs = fs.getFileBlockLocations(srcPath, 0, cellSize);
@@ -112,7 +124,8 @@ public class TestReadStripedFileWithMissingBlocks {
// make sure there are missing block locations
BlockLocation[] newLocs = fs.getFileBlockLocations(srcPath, 0, cellSize);
- Assert.assertTrue(newLocs[0].getNames().length < locs[0].getNames().length);
+ Assert.assertTrue(
+ newLocs[0].getNames().length < locs[0].getNames().length);
byte[] smallBuf = new byte[1024];
byte[] largeBuf = new byte[fileLength + 100];
@@ -120,10 +133,18 @@ public class TestReadStripedFileWithMissingBlocks {
blockGroupSize);
StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
smallBuf);
- StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+ StripedFileTestUtil
+ .verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+ restartDeadDataNodes();
+ }
- // delete the file
- fs.delete(srcPath, true);
+ private void restartDeadDataNodes() throws IOException {
+ DatanodeInfo[] deadNodes = dfsClient
+ .datanodeReport(DatanodeReportType.DEAD);
+ for (DatanodeInfo dnInfo : deadNodes) {
+ cluster.restartDataNode(dnInfo.getXferAddr());
+ }
+ cluster.triggerHeartbeats();
}
private void stopDataNodes(BlockLocation[] locs, int[] datanodes)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org