You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/10/27 11:31:34 UTC
[1/3] hadoop git commit: HDFS-8492. DN should notify NN when client
requests a missing block (Contributed by Walter Su)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 14ce42a8e -> 144f7a9e0
refs/heads/branch-2.8 6d9b6fac8 -> 7279ce831
refs/heads/trunk 4e403def8 -> 1cf6ec4ad
HDFS-8492. DN should notify NN when client requests a missing block (Contributed by Walter Su)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cf6ec4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cf6ec4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cf6ec4a
Branch: refs/heads/trunk
Commit: 1cf6ec4ad4e1f4ea71f912923b5e8627b61ef482
Parents: 4e403de
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Oct 27 16:44:00 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Oct 27 16:44:00 2016 +0530
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockSender.java | 7 ++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
.../apache/hadoop/hdfs/TestDatanodeReport.java | 38 ++++++++++++++++++--
3 files changed, 44 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf6ec4a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index bfc53f1..a1b1f86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -36,6 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -322,6 +323,12 @@ class BlockSender implements java.io.Closeable {
} else {
LOG.warn("Could not find metadata file for " + block);
}
+ } catch (FileNotFoundException e) {
+ // The replica is on its volume map but not on disk
+ datanode.notifyNamenodeDeletedBlock(block, replica.getStorageUuid());
+ datanode.data.invalidate(block.getBlockPoolId(),
+ new Block[]{block.getLocalBlock()});
+ throw e;
} finally {
if (!keepMetaInOpen) {
IOUtils.closeStream(metaIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf6ec4a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index c61fc57..abeda2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -738,7 +738,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaInfo getBlockReplica(String bpid, long blockId) throws IOException {
ReplicaInfo r = validateBlockFile(bpid, blockId);
if (r == null) {
- throw new IOException("BlockId " + blockId + " is not valid.");
+ throw new FileNotFoundException("BlockId " + blockId + " is not valid.");
}
return r;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf6ec4a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
index f23e531..fea377f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
@@ -29,16 +29,20 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeAdminProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager;
import org.apache.hadoop.hdfs.server.blockmanagement.HostConfigManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.util.HostsFileWriter;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -144,7 +148,37 @@ public class TestDatanodeReport {
cluster.shutdown();
}
}
-
+
+ @Test
+ public void testDatanodeReportMissingBlock() throws Exception {
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 1);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_OF_DATANODES).build();
+ try {
+ // wait until the cluster is up
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path p = new Path("/testDatanodeReportMissingBlock");
+ DFSTestUtil.writeFile(fs, p, new String("testdata"));
+ LocatedBlock lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(3, lb.getLocations().length);
+ ExtendedBlock b = lb.getBlock();
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(b);
+ try {
+ DFSTestUtil.readFile(fs, p);
+ Assert.fail("Must throw exception as the block doesn't exists on disk");
+ } catch (IOException e) {
+ // all bad datanodes
+ }
+ cluster.triggerHeartbeats(); // IBR delete ack
+ lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(0, lb.getLocations().length);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
final static Comparator<StorageReport> CMP = new Comparator<StorageReport>() {
@Override
public int compare(StorageReport left, StorageReport right) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HDFS-8492. DN should notify NN when client
requests a missing block (Contributed by Walter Su)
Posted by vi...@apache.org.
HDFS-8492. DN should notify NN when client requests a missing block (Contributed by Walter Su)
(cherry picked from commit 1cf6ec4ad4e1f4ea71f912923b5e8627b61ef482)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/144f7a9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/144f7a9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/144f7a9e
Branch: refs/heads/branch-2
Commit: 144f7a9e0eb3084eaaff7ab0bee1f5943e16d136
Parents: 14ce42a
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Oct 27 16:44:00 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Oct 27 16:46:09 2016 +0530
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockSender.java | 7 ++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
.../apache/hadoop/hdfs/TestDatanodeReport.java | 38 ++++++++++++++++++--
3 files changed, 44 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f7a9e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 767d36e..4d3a45a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -36,6 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
@@ -322,6 +323,12 @@ class BlockSender implements java.io.Closeable {
} else {
LOG.warn("Could not find metadata file for " + block);
}
+ } catch (FileNotFoundException e) {
+ // The replica is on its volume map but not on disk
+ datanode.notifyNamenodeDeletedBlock(block, replica.getStorageUuid());
+ datanode.data.invalidate(block.getBlockPoolId(),
+ new Block[]{block.getLocalBlock()});
+ throw e;
} finally {
if (!keepMetaInOpen) {
IOUtils.closeStream(metaIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f7a9e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 1322e24..42256ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -754,7 +754,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File getBlockFile(String bpid, long blockId) throws IOException {
File f = validateBlockFile(bpid, blockId);
if(f == null) {
- throw new IOException("BlockId " + blockId + " is not valid.");
+ throw new FileNotFoundException("BlockId " + blockId + " is not valid.");
}
return f;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144f7a9e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
index f23e531..fea377f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
@@ -29,16 +29,20 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeAdminProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager;
import org.apache.hadoop.hdfs.server.blockmanagement.HostConfigManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.util.HostsFileWriter;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -144,7 +148,37 @@ public class TestDatanodeReport {
cluster.shutdown();
}
}
-
+
+ @Test
+ public void testDatanodeReportMissingBlock() throws Exception {
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 1);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_OF_DATANODES).build();
+ try {
+ // wait until the cluster is up
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path p = new Path("/testDatanodeReportMissingBlock");
+ DFSTestUtil.writeFile(fs, p, new String("testdata"));
+ LocatedBlock lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(3, lb.getLocations().length);
+ ExtendedBlock b = lb.getBlock();
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(b);
+ try {
+ DFSTestUtil.readFile(fs, p);
+ Assert.fail("Must throw exception as the block doesn't exists on disk");
+ } catch (IOException e) {
+ // all bad datanodes
+ }
+ cluster.triggerHeartbeats(); // IBR delete ack
+ lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(0, lb.getLocations().length);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
final static Comparator<StorageReport> CMP = new Comparator<StorageReport>() {
@Override
public int compare(StorageReport left, StorageReport right) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HDFS-8492. DN should notify NN when client
requests a missing block (Contributed by Walter Su)
Posted by vi...@apache.org.
HDFS-8492. DN should notify NN when client requests a missing block (Contributed by Walter Su)
(cherry picked from commit 1cf6ec4ad4e1f4ea71f912923b5e8627b61ef482)
(cherry picked from commit 144f7a9e0eb3084eaaff7ab0bee1f5943e16d136)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7279ce83
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7279ce83
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7279ce83
Branch: refs/heads/branch-2.8
Commit: 7279ce831d467ff480131b411e9e5674ff902b4a
Parents: 6d9b6fa
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Oct 27 16:44:00 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Oct 27 16:56:25 2016 +0530
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockSender.java | 7 ++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
.../apache/hadoop/hdfs/TestDatanodeReport.java | 37 +++++++++++++++++++-
3 files changed, 44 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7279ce83/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 767d36e..4d3a45a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -36,6 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
@@ -322,6 +323,12 @@ class BlockSender implements java.io.Closeable {
} else {
LOG.warn("Could not find metadata file for " + block);
}
+ } catch (FileNotFoundException e) {
+ // The replica is on its volume map but not on disk
+ datanode.notifyNamenodeDeletedBlock(block, replica.getStorageUuid());
+ datanode.data.invalidate(block.getBlockPoolId(),
+ new Block[]{block.getLocalBlock()});
+ throw e;
} finally {
if (!keepMetaInOpen) {
IOUtils.closeStream(metaIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7279ce83/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 47f8f72..0b2c7b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -752,7 +752,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File getBlockFile(String bpid, long blockId) throws IOException {
File f = validateBlockFile(bpid, blockId);
if(f == null) {
- throw new IOException("BlockId " + blockId + " is not valid.");
+ throw new FileNotFoundException("BlockId " + blockId + " is not valid.");
}
return f;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7279ce83/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
index 1e6db21..8138298 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
@@ -29,11 +29,16 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -89,7 +94,37 @@ public class TestDatanodeReport {
cluster.shutdown();
}
}
-
+
+ @Test
+ public void testDatanodeReportMissingBlock() throws Exception {
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 1);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_OF_DATANODES).build();
+ try {
+ // wait until the cluster is up
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path p = new Path("/testDatanodeReportMissingBlock");
+ DFSTestUtil.writeFile(fs, p, new String("testdata"));
+ LocatedBlock lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(3, lb.getLocations().length);
+ ExtendedBlock b = lb.getBlock();
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(b);
+ try {
+ DFSTestUtil.readFile(fs, p);
+ Assert.fail("Must throw exception as the block doesn't exists on disk");
+ } catch (IOException e) {
+ // all bad datanodes
+ }
+ cluster.triggerHeartbeats(); // IBR delete ack
+ lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
+ assertEquals(0, lb.getLocations().length);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
final static Comparator<StorageReport> CMP = new Comparator<StorageReport>() {
@Override
public int compare(StorageReport left, StorageReport right) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org