You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/09/02 16:54:25 UTC
[hadoop-ozone] 06/33: HDDS-4094. Support byte-level write in Freon
HadoopFsGenerator (#1310)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit cfdaaa1daa9aaf689006ebce3a2b02aa345df3dd
Author: Elek, Márton <el...@users.noreply.github.com>
AuthorDate: Mon Aug 24 15:14:41 2020 +0200
HDDS-4094. Support byte-level write in Freon HadoopFsGenerator (#1310)
---
.../hadoop/ozone/freon/ContentGenerator.java | 31 +++++++-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +++-
.../hadoop/ozone/freon/TestContentGenerator.java | 82 ++++++++++++++++++++++
3 files changed, 119 insertions(+), 6 deletions(-)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
index c6ec60e..542634c 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.RandomStringUtils;
/**
@@ -38,15 +39,25 @@ public class ContentGenerator {
*/
private int bufferSize;
+ /**
+ * Number of bytes to write in one call.
+ * <p>
+ * Should be no larger than the bufferSize.
+ */
+ private final int copyBufferSize;
+
private final byte[] buffer;
ContentGenerator(long keySize, int bufferSize) {
+ this(keySize, bufferSize, bufferSize);
+ }
+
+ ContentGenerator(long keySize, int bufferSize, int copyBufferSize) {
this.keySize = keySize;
this.bufferSize = bufferSize;
-
+ this.copyBufferSize = copyBufferSize;
buffer = RandomStringUtils.randomAscii(bufferSize)
.getBytes(StandardCharsets.UTF_8);
-
}
/**
@@ -56,7 +67,21 @@ public class ContentGenerator {
for (long nrRemaining = keySize;
nrRemaining > 0; nrRemaining -= bufferSize) {
int curSize = (int) Math.min(bufferSize, nrRemaining);
- outputStream.write(buffer, 0, curSize);
+ if (copyBufferSize == 1) {
+ for (int i = 0; i < curSize; i++) {
+ outputStream.write(buffer[i]);
+ }
+ } else {
+ for (int i = 0; i < curSize; i += copyBufferSize) {
+ outputStream.write(buffer, i,
+ Math.min(copyBufferSize, curSize - i));
+ }
+ }
}
}
+
+ @VisibleForTesting
+ byte[] getBuffer() {
+ return buffer;
+ }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
index 548f829..925ba7d 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
@@ -57,10 +57,15 @@ public class HadoopFsGenerator extends BaseFreonGenerator
private int fileSize;
@Option(names = {"--buffer"},
- description = "Size of buffer used to generated the key content.",
- defaultValue = "4096")
+ description = "Size of buffer used store the generated key content",
+ defaultValue = "10240")
private int bufferSize;
+ @Option(names = {"--copy-buffer"},
+ description = "Size of bytes written to the output in one operation",
+ defaultValue = "4096")
+ private int copyBufferSize;
+
private ContentGenerator contentGenerator;
private Timer timer;
@@ -76,7 +81,8 @@ public class HadoopFsGenerator extends BaseFreonGenerator
fileSystem = FileSystem.get(URI.create(rootPath), configuration);
- contentGenerator = new ContentGenerator(fileSize, bufferSize);
+ contentGenerator =
+ new ContentGenerator(fileSize, bufferSize, copyBufferSize);
timer = getMetrics().timer("file-create");
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java
new file mode 100644
index 0000000..d61be3a
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java
@@ -0,0 +1,82 @@
+package org.apache.hadoop.ozone.freon;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the ContentGenerator class of Freon.
+ */
+public class TestContentGenerator {
+
+ @Test
+ public void writeWrite() throws IOException {
+ ContentGenerator generator = new ContentGenerator(1024, 1024);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ generator.write(output);
+ Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray());
+ }
+
+ @Test
+ public void writeWithSmallerBuffers() throws IOException {
+ ContentGenerator generator = new ContentGenerator(10000, 1024, 3);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ generator.write(baos);
+
+ Assert.assertEquals(10000, baos.toByteArray().length);
+ }
+
+ @Test
+ public void writeWithByteLevelWrite() throws IOException {
+ ContentGenerator generator = new ContentGenerator(1024, 1024, 1);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ generator.write(output);
+ Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray());
+ }
+
+ @Test
+ public void writeWithSmallBuffer() throws IOException {
+ ContentGenerator generator = new ContentGenerator(1024, 1024, 10);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ generator.write(output);
+ Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray());
+ }
+
+ @Test
+ public void writeWithDistinctSizes() throws IOException {
+ ContentGenerator generator = new ContentGenerator(20, 8, 3);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ generator.write(output);
+
+ byte[] expected = new byte[20];
+ byte[] buffer = generator.getBuffer();
+ System.arraycopy(buffer, 0, expected, 0, buffer.length);
+ System.arraycopy(buffer, 0, expected, 8, buffer.length);
+ System.arraycopy(buffer, 0, expected, 16, 4);
+ Assert.assertArrayEquals(expected, output.toByteArray());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org