You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ck...@apache.org on 2023/02/09 02:17:38 UTC
[ozone] branch master updated: HDDS-7923. [EC] Reconstruction is failing with IndexOutOfBoundsException (#4258)
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b5230a2756 HDDS-7923. [EC] Reconstruction is failing with IndexOutOfBoundsException (#4258)
b5230a2756 is described below
commit b5230a2756cf06ab4cfb59c926bd82583fa6857d
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Wed Feb 8 18:17:33 2023 -0800
HDDS-7923. [EC] Reconstruction is failing with IndexOutOfBoundsException (#4258)
---
.../hdds/scm/storage/ECBlockOutputStream.java | 3 ++-
.../hdds/scm/storage/TestContainerCommandsEC.java | 27 ++++++++++++++--------
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
index c8ea45b194..1fb4b1dfb1 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
@@ -117,7 +117,8 @@ public class ECBlockOutputStream extends BlockOutputStream {
continue;
}
List<ChunkInfo> chunks = bd.getChunks();
- if (chunks != null && chunks.get(0).hasStripeChecksum()) {
+ if (chunks != null && chunks.size() > 0 && chunks.get(0)
+ .hasStripeChecksum()) {
checksumBlockData = bd;
break;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
index 9b7b3d90c3..68dfd6fbb7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
@@ -349,9 +349,16 @@ public class TestContainerCommandsEC {
@MethodSource("recoverableMissingIndexes")
void testECReconstructionCoordinatorWith(List<Integer> missingIndexes)
throws Exception {
- testECReconstructionCoordinator(missingIndexes);
+ testECReconstructionCoordinator(missingIndexes, 3);
}
+ @Test
+ void testECReconstructionWithPartialStripe()
+ throws Exception {
+ testECReconstructionCoordinator(ImmutableList.of(4, 5), 1);
+ }
+
+
static Stream<List<Integer>> recoverableMissingIndexes() {
return Stream
.concat(IntStream.rangeClosed(1, 5).mapToObj(ImmutableList::of), Stream
@@ -367,7 +374,7 @@ public class TestContainerCommandsEC {
public void testECReconstructionCoordinatorWithMissingIndexes135() {
InsufficientLocationsException exception =
Assert.assertThrows(InsufficientLocationsException.class, () -> {
- testECReconstructionCoordinator(ImmutableList.of(1, 3, 5));
+ testECReconstructionCoordinator(ImmutableList.of(1, 3, 5), 3);
});
String expectedMessage =
@@ -377,8 +384,8 @@ public class TestContainerCommandsEC {
Assert.assertEquals(expectedMessage, actualMessage);
}
- private void testECReconstructionCoordinator(List<Integer> missingIndexes)
- throws Exception {
+ private void testECReconstructionCoordinator(List<Integer> missingIndexes,
+ int numInputChunks) throws Exception {
ObjectStore objectStore = rpcClient.getObjectStore();
String keyString = UUID.randomUUID().toString();
String volumeName = UUID.randomUUID().toString();
@@ -389,7 +396,7 @@ public class TestContainerCommandsEC {
OzoneBucket bucket = volume.getBucket(bucketName);
XceiverClientManager xceiverClientManager =
new XceiverClientManager(config);
- createKeyAndWriteData(keyString, bucket);
+ createKeyAndWriteData(keyString, bucket, numInputChunks);
ECReconstructionCoordinator coordinator =
new ECReconstructionCoordinator(config, certClient,
null, ECReconstructionMetrics.create());
@@ -499,15 +506,15 @@ public class TestContainerCommandsEC {
Assertions.assertEquals(metrics.getReconstructionTotal(), 1L);
}
- private void createKeyAndWriteData(String keyString, OzoneBucket bucket)
- throws IOException {
- for (int i = 0; i < EC_DATA; i++) {
+ private void createKeyAndWriteData(String keyString, OzoneBucket bucket,
+ int numChunks) throws IOException {
+ for (int i = 0; i < numChunks; i++) {
inputChunks[i] = getBytesWith(i + 1, EC_CHUNK_SIZE);
}
try (OzoneOutputStream out = bucket.createKey(keyString, 4096,
new ECReplicationConfig(3, 2, EcCodec.RS, 1024), new HashMap<>())) {
Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream);
- for (int i = 0; i < inputChunks.length; i++) {
+ for (int i = 0; i < numChunks; i++) {
out.write(inputChunks[i]);
}
}
@@ -525,7 +532,7 @@ public class TestContainerCommandsEC {
objectStore.getVolume(volumeName).createBucket(bucketName);
OzoneVolume volume = objectStore.getVolume(volumeName);
OzoneBucket bucket = volume.getBucket(bucketName);
- createKeyAndWriteData(keyString, bucket);
+ createKeyAndWriteData(keyString, bucket, 3);
OzoneKeyDetails key = bucket.getKey(keyString);
long conID = key.getOzoneKeyLocations().get(0).getContainerID();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org