You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2021/10/07 00:15:44 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-5822 : Writing a large
buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 7a67f63 HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
7a67f63 is described below
commit 7a67f63429d82d4d7ccba6bd62e2955d29939ba6
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Wed Oct 6 17:15:31 2021 -0700
HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
---
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 2 +
.../hadoop/ozone/client/TestOzoneECClient.java | 90 +++++++++++++++++-----
.../ozone/client/rpc/TestECKeyOutputStream.java | 85 +++++++++++++++-----
3 files changed, 139 insertions(+), 38 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
index b5ab25c..61107b0 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
@@ -199,6 +199,8 @@ public class ECKeyOutputStream extends KeyOutputStream {
int remLen = len - currentWriterChunkLenToWrite;
int iters = remLen / ecChunkSize;
int lastCellSize = remLen % ecChunkSize;
+ off += currentWriterChunkLenToWrite;
+
while (iters > 0) {
pos = handleDataWrite(blockOutputStreamEntryPool.getCurrIdx(), b, off,
ecChunkSize, true);
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
index 5fe11a3..daf1789 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
@@ -212,6 +212,69 @@ public class TestOzoneECClient {
}
@Test
+ public void test4ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(4);
+ }
+
+ // Test random number of chunks in single write op.
+ @Test
+ public void test5ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(5);
+ }
+
+ @Test
+ public void test6ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(6);
+ }
+
+ @Test
+ public void test7ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(7);
+ }
+
+ @Test
+ public void test9ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(9);
+ }
+
+ @Test
+ public void test10ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(10);
+ }
+
+ @Test
+ public void test12ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(12);
+ }
+
+ public void testMultipleChunksInSingleWriteOp(int numChunks)
+ throws IOException {
+ byte[] inputData = new byte[numChunks * chunkSize];
+ for (int i = 0; i < numChunks; i++) {
+ int start = (i * chunkSize);
+ Arrays.fill(inputData, start, start + chunkSize - 1,
+ String.valueOf(i % 9).getBytes(UTF_8)[0]);
+ }
+ final OzoneBucket bucket = writeIntoECKey(inputData, keyName,
+ new DefaultReplicationConfig(ReplicationType.EC,
+ new ECReplicationConfig(dataBlocks, parityBlocks,
+ ECReplicationConfig.EcCodec.RS, chunkSize)));
+ OzoneKey key = bucket.getKey(keyName);
+ validateContent(inputData, bucket, key);
+ }
+
+ private void validateContent(byte[] inputData, OzoneBucket bucket,
+ OzoneKey key) throws IOException {
+ Assert.assertEquals(keyName, key.getName());
+ try (OzoneInputStream is = bucket.readKey(keyName)) {
+ byte[] fileContent = new byte[inputData.length];
+ Assert.assertEquals(inputData.length, is.read(fileContent));
+ Assert.assertEquals(new String(inputData, UTF_8),
+ new String(fileContent, UTF_8));
+ }
+ }
+
+ @Test
public void testCommitKeyInfo()
throws IOException {
final OzoneBucket bucket = writeIntoECKey(inputChunks, keyName,
@@ -249,15 +312,8 @@ public class TestOzoneECClient {
out.write(inputChunks[0][i]);
}
}
-
OzoneKey key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
- try (OzoneInputStream is = bucket.readKey(keyName)) {
- byte[] fileContent = new byte[1024];
- Assert.assertEquals(inputChunks[0].length, is.read(fileContent));
- Assert.assertEquals(new String(inputChunks[0], UTF_8),
- new String(fileContent, UTF_8));
- }
+ validateContent(inputChunks[0], bucket, key);
}
@Test
@@ -275,16 +331,9 @@ public class TestOzoneECClient {
out.write(inputChunks[0][i]);
}
}
-
OzoneKey key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
- try (OzoneInputStream is = bucket.readKey(keyName)) {
- byte[] fileContent = new byte[1023];
- Assert.assertEquals(inputChunks[0].length - 1, is.read(fileContent));
- Assert.assertEquals(
- new String(Arrays.copyOf(inputChunks[0], inputChunks[0].length - 1),
- UTF_8), new String(fileContent, UTF_8));
- }
+ validateContent(Arrays.copyOf(inputChunks[0], inputChunks[0].length - 1),
+ bucket, key);
}
@Test
@@ -326,6 +375,11 @@ public class TestOzoneECClient {
}
}
+ private OzoneBucket writeIntoECKey(byte[] data, String key,
+ DefaultReplicationConfig defaultReplicationConfig) throws IOException {
+ return writeIntoECKey(new byte[][] {data}, key, defaultReplicationConfig);
+ }
+
private OzoneBucket writeIntoECKey(byte[][] chunks, String key,
DefaultReplicationConfig defaultReplicationConfig) throws IOException {
store.createVolume(volumeName);
@@ -339,7 +393,7 @@ public class TestOzoneECClient {
}
OzoneBucket bucket = volume.getBucket(bucketName);
- try (OzoneOutputStream out = bucket.createKey(key, 2000,
+ try (OzoneOutputStream out = bucket.createKey(key, 4096,
new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
chunkSize), new HashMap<>())) {
for (int i = 0; i < chunks.length; i++) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
index d6a0ea8..84ccd44 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry;
import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
@@ -47,6 +48,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -150,17 +152,7 @@ public class TestECKeyOutputStream {
@Test
public void testCreateKeyWithBucketDefaults() throws Exception {
- String myBucket = UUID.randomUUID().toString();
- OzoneVolume volume = objectStore.getVolume(volumeName);
- final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
- bucketArgs.setDefaultReplicationConfig(
- new DefaultReplicationConfig(ReplicationType.EC,
- new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
- chunkSize)));
-
- volume.createBucket(myBucket, bucketArgs.build());
- OzoneBucket bucket = volume.getBucket(myBucket);
-
+ OzoneBucket bucket = getOzoneBucket();
try (OzoneOutputStream out = bucket.createKey(keyString, 2000)) {
Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream);
for (int i = 0; i < inputChunks.length; i++) {
@@ -179,6 +171,67 @@ public class TestECKeyOutputStream {
@Test
public void testCreateRatisKeyAndWithECBucketDefaults() throws Exception {
+ OzoneBucket bucket = getOzoneBucket();
+ try (OzoneOutputStream out = bucket
+ .createKey("testCreateRatisKeyAndWithECBucketDefaults", 2000,
+ new RatisReplicationConfig("3"), new HashMap<>())) {
+ Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream);
+ for (int i = 0; i < inputChunks.length; i++) {
+ out.write(inputChunks[i]);
+ }
+ }
+ }
+
+ @Test
+ public void test13ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(13);
+ }
+
+ @Test
+ public void test15ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(15);
+ }
+
+ @Test
+ public void test20ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(20);
+ }
+
+ @Test
+ public void test21ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(21);
+ }
+
+ public void testMultipleChunksInSingleWriteOp(int numChunks)
+ throws IOException {
+ byte[] inputData = new byte[numChunks * chunkSize];
+ for (int i = 0; i < numChunks; i++) {
+ int start = (i * chunkSize);
+ Arrays.fill(inputData, start, start + chunkSize - 1,
+ String.valueOf(i % 9).getBytes(UTF_8)[0]);
+ }
+ final OzoneBucket bucket = getOzoneBucket();
+ String keyName = "testMultipleChunksInSingleWriteOp" + numChunks;
+ try (OzoneOutputStream out = bucket.createKey(keyName, 4096,
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize), new HashMap<>())) {
+ out.write(inputData);
+ }
+
+ validateContent(inputData, bucket, bucket.getKey(keyName));
+ }
+
+ private void validateContent(byte[] inputData, OzoneBucket bucket,
+ OzoneKey key) throws IOException {
+ try (OzoneInputStream is = bucket.readKey(key.getName())) {
+ byte[] fileContent = new byte[inputData.length];
+ Assert.assertEquals(inputData.length, is.read(fileContent));
+ Assert.assertEquals(new String(inputData, UTF_8),
+ new String(fileContent, UTF_8));
+ }
+ }
+
+ private OzoneBucket getOzoneBucket() throws IOException {
String myBucket = UUID.randomUUID().toString();
OzoneVolume volume = objectStore.getVolume(volumeName);
final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
@@ -188,15 +241,7 @@ public class TestECKeyOutputStream {
chunkSize)));
volume.createBucket(myBucket, bucketArgs.build());
- OzoneBucket bucket = volume.getBucket(myBucket);
- try (OzoneOutputStream out = bucket
- .createKey("testCreateRatisKeyAndWithECBucketDefaults", 2000,
- new RatisReplicationConfig("3"), new HashMap<>())) {
- Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream);
- for (int i = 0; i < inputChunks.length; i++) {
- out.write(inputChunks[i]);
- }
- }
+ return volume.getBucket(myBucket);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org