You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2022/06/01 05:55:45 UTC
[ozone] branch master updated: HDDS-6808. EC: Fix datanode exclusion check in client (#3460)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new af3a80d041 HDDS-6808. EC: Fix datanode exclusion check in client (#3460)
af3a80d041 is described below
commit af3a80d041b0cd3892e7505b6650e3131998d695
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Wed Jun 1 13:55:39 2022 +0800
HDDS-6808. EC: Fix datanode exclusion check in client (#3460)
---
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 3 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 10 ++--
.../ozone/client/MockXceiverClientFactory.java | 60 ++++++++++++-------
.../hadoop/ozone/client/MockXceiverClientSpi.java | 3 +
.../hadoop/ozone/client/TestOzoneECClient.java | 67 +++++++++++++++++++++-
5 files changed, 114 insertions(+), 29 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
index df279d8743..04daa519c9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
@@ -249,7 +249,8 @@ public final class ECKeyOutputStream extends KeyOutputStream {
// If the failure is NOT caused by other reasons (e.g. container full),
// we assume it is caused by DN failure and exclude the failed DN.
failedStreams.stream()
- .filter(s -> !checkIfContainerToExclude(s.getIoException()))
+ .filter(s -> !checkIfContainerToExclude(
+ HddsClientUtils.checkForException(s.getIoException())))
.forEach(s -> blockOutputStreamEntryPool.getExcludeList()
.addDatanode(s.getDatanodeDetails()));
}
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
index 1f29425e3d..b494fdf3cd 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
@@ -40,10 +40,10 @@ public class MockDatanodeStorage {
private final Map<String, ByteString> data = new HashMap<>();
- private boolean failed = false;
+ private IOException exception = null;
- public void setStorageFailed() {
- this.failed = true;
+ public void setStorageFailed(IOException reason) {
+ this.exception = reason;
}
public void putBlock(DatanodeBlockID blockID, BlockData blockData) {
@@ -57,8 +57,8 @@ public class MockDatanodeStorage {
public void writeChunk(
DatanodeBlockID blockID,
ChunkInfo chunkInfo, ByteString bytes) throws IOException {
- if (failed) {
- throw new IOException("This storage was marked as failed.");
+ if (exception != null) {
+ throw exception;
}
data.put(createKey(blockID, chunkInfo),
ByteString.copyFrom(bytes.toByteArray()));
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientFactory.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientFactory.java
index fbcc1531c6..8c5fa76291 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientFactory.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientFactory.java
@@ -23,11 +23,14 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Factory to create the mock datanode clients.
@@ -37,30 +40,41 @@ public class MockXceiverClientFactory
private final Map<DatanodeDetails, MockDatanodeStorage> storage =
new HashMap<>();
- private List<DatanodeDetails> pendingToFailNodes = new ArrayList<>();
+ private final Map<IOException, Set<DatanodeDetails>> pendingDNFailures =
+ new HashMap<>();
public void setFailedStorages(List<DatanodeDetails> failedStorages) {
- List<DatanodeDetails> remainingFailNodes = new ArrayList<>();
- for (int i = 0; i < failedStorages.size(); i++) {
- DatanodeDetails failedDN = failedStorages.get(i);
- boolean isCurrentNodeMarked = false;
- final Iterator<Map.Entry<DatanodeDetails, MockDatanodeStorage>> iterator =
- storage.entrySet().iterator();
- while (iterator.hasNext()) {
- final Map.Entry<DatanodeDetails, MockDatanodeStorage> next =
- iterator.next();
- if (next.getKey().equals(failedDN)) {
- final MockDatanodeStorage value = next.getValue();
- value.setStorageFailed();
- isCurrentNodeMarked = true;
- }
- }
- if (!isCurrentNodeMarked) {
- //This node does not initialized by client yet.
- remainingFailNodes.add(failedDN);
+ mockStorageFailure(failedStorages,
+ new IOException("This storage was marked as failed."));
+ }
+
+ public void mockStorageFailure(Collection<DatanodeDetails> datanodes,
+ IOException reason) {
+ if (!pendingDNFailures.containsKey(reason)) {
+ pendingDNFailures.put(reason, new HashSet<>());
+ }
+ pendingDNFailures.get(reason).addAll(datanodes);
+ mockStorageFailure(reason);
+ }
+
+ private void mockStorageFailure(IOException reason) {
+
+ Collection<DatanodeDetails> datanodes =
+ pendingDNFailures.getOrDefault(reason, Collections.emptySet());
+
+ Iterator<DatanodeDetails> iterator = datanodes.iterator();
+ while (iterator.hasNext()) {
+ DatanodeDetails dn = iterator.next();
+ MockDatanodeStorage mockDN = storage.get(dn);
+ if (mockDN != null) {
+ mockDN.setStorageFailed(reason);
+ iterator.remove();
}
}
- this.pendingToFailNodes = remainingFailNodes;
+
+ if (datanodes.isEmpty()) {
+ pendingDNFailures.remove(reason);
+ }
}
@Override
@@ -76,7 +90,9 @@ public class MockXceiverClientFactory
.computeIfAbsent(pipeline.getFirstNode(),
r -> new MockDatanodeStorage()));
// Incase if this node already set to mark as failed.
- setFailedStorages(this.pendingToFailNodes);
+ for (IOException reason : pendingDNFailures.keySet()) {
+ mockStorageFailure(reason);
+ }
return mockXceiverClientSpi;
}
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientSpi.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientSpi.java
index 4dfe966a90..e468f16f95 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientSpi.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockXceiverClientSpi.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunk
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import java.io.IOException;
@@ -86,6 +87,8 @@ public class MockXceiverClientSpi extends XceiverClientSpi {
r -> {
try {
return r.setWriteChunk(writeChunk(request.getWriteChunk()));
+ } catch (ContainerNotOpenException e) {
+ return r.setResult(Result.CLOSED_CONTAINER_IO);
} catch (IOException e) {
return r.setResult(Result.IO_EXCEPTION);
}
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
index e9ba156b31..024daa891d 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -61,6 +62,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -814,6 +816,69 @@ public class TestOzoneECClient {
}
}
+ @Test
+ public void testExcludeOnDNFailure() throws IOException {
+ testExcludeFailedDN(IntStream.range(0, 5), IntStream.empty());
+ }
+
+ @Test
+ public void testExcludeOnDNClosed() throws IOException {
+ testExcludeFailedDN(IntStream.empty(), IntStream.range(0, 5));
+ }
+
+ @Test
+ public void testExcludeOnDNMixed() throws IOException {
+ testExcludeFailedDN(IntStream.range(0, 3), IntStream.range(3, 5));
+ }
+
+ private void testExcludeFailedDN(IntStream failedDNIndex,
+ IntStream closedDNIndex) throws IOException {
+ close();
+ OzoneConfiguration con = new OzoneConfiguration();
+ MultiNodePipelineBlockAllocator blkAllocator =
+ new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks, 10);
+ createNewClient(con, blkAllocator);
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ ECReplicationConfig repConfig = new ECReplicationConfig(
+ dataBlocks, parityBlocks, ECReplicationConfig.EcCodec.RS, chunkSize);
+
+ try (OzoneOutputStream out = bucket.createKey(keyName,
+ (long) dataBlocks * chunkSize, repConfig, new HashMap<>())) {
+
+ Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream);
+ ECKeyOutputStream ecKeyOut = (ECKeyOutputStream) out.getOutputStream();
+
+ List<HddsProtos.DatanodeDetailsProto> dns = blkAllocator.getClusterDns();
+
+ // Then let's mark datanodes with closed container
+ List<DatanodeDetails> closedDNs = closedDNIndex
+ .mapToObj(i -> DatanodeDetails.getFromProtoBuf(dns.get(i)))
+ .collect(Collectors.toList());
+ ((MockXceiverClientFactory) factoryStub).mockStorageFailure(closedDNs,
+ new ContainerNotOpenException("Mocked"));
+
+ // Then let's mark failed datanodes
+ List<DatanodeDetails> failedDNs = failedDNIndex
+ .mapToObj(i -> DatanodeDetails.getFromProtoBuf(dns.get(i)))
+ .collect(Collectors.toList());
+ ((MockXceiverClientFactory) factoryStub).setFailedStorages(failedDNs);
+
+ for (int i = 0; i < dataBlocks; i++) {
+ out.write(inputChunks[i % dataBlocks]);
+ }
+
+ // Assert excludeList only includes failedDNs
+ Assert.assertArrayEquals(failedDNs.toArray(new DatanodeDetails[0]),
+ ecKeyOut.getExcludeList().getDatanodes()
+ .toArray(new DatanodeDetails[0]));
+ }
+ }
+
@Test
public void testLargeWriteOfMultipleStripesWithStripeFailure()
throws IOException {
@@ -1110,4 +1175,4 @@ public class TestOzoneECClient {
}
return locationInfoList;
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org