You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/13 06:52:21 UTC
[hadoop-ozone] 01/01: HDDS-2271. Avoid buffer copying in
KeyValueHandler.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2271
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 38ec79e26f38bc690950a9102012386e46bcd3df
Author: Tsz Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Wed Oct 9 17:02:48 2019 +0800
HDDS-2271. Avoid buffer copying in KeyValueHandler.
---
.../apache/hadoop/ozone/container/common/helpers/BlockData.java | 5 +++++
.../hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java | 3 +--
.../apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java | 9 ++++-----
.../hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java | 4 ++--
4 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
index e0cac8b..ceefa90 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
@@ -117,6 +117,11 @@ public class BlockData {
return builder.build();
}
+ public int getSerializedSize() {
+ //TODO: the serialized size may be computed without creating the proto.
+ return getProtoBufMessage().getSerializedSize();
+ }
+
/**
* Adds metadata.
*
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
index a4bd376..b8f5f11 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
@@ -267,8 +267,7 @@ public class KeyValueContainerCheck {
ByteString expected = cData.getChecksums().get(i);
ByteString actual = cal.computeChecksum(buffer, 0, v)
.getChecksums().get(0);
- if (!Arrays.equals(expected.toByteArray(),
- actual.toByteArray())) {
+ if (!expected.equals(actual)) {
throw new OzoneChecksumException(String
.format("Inconsistent read for chunk=%s len=%d expected" +
" checksum %s actual checksum %s for block %s",
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index bc41883..995b913 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -387,7 +387,6 @@ public class KeyValueHandler extends Handler {
ContainerCommandRequestProto request, KeyValueContainer kvContainer,
DispatcherContext dispatcherContext) {
- long blockLength;
if (!request.hasPutBlock()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Malformed Put Key request. trace ID: {}",
@@ -406,7 +405,7 @@ public class KeyValueHandler extends Handler {
long bcsId =
dispatcherContext == null ? 0 : dispatcherContext.getLogIndex();
blockData.setBlockCommitSequenceId(bcsId);
- long numBytes = blockData.getProtoBufMessage().toByteArray().length;
+ final long numBytes = blockData.getSerializedSize();
blockManager.putBlock(kvContainer, blockData);
metrics.incContainerBytesStats(Type.PutBlock, numBytes);
} catch (StorageContainerException ex) {
@@ -447,7 +446,7 @@ public class KeyValueHandler extends Handler {
BlockID blockID = BlockID.getFromProtobuf(
request.getGetBlock().getBlockID());
responseData = blockManager.getBlock(kvContainer, blockID);
- long numBytes = responseData.getProtoBufMessage().toByteArray().length;
+ final long numBytes = responseData.getSerializedSize();
metrics.incContainerBytesStats(Type.GetBlock, numBytes);
} catch (StorageContainerException ex) {
@@ -818,8 +817,8 @@ public class KeyValueHandler extends Handler {
chunkInfo = chunk;
}
metrics.incContainerBytesStats(Type.GetSmallFile, dataBuf.size());
- return SmallFileUtils.getGetSmallFileResponseSuccess(request, dataBuf
- .toByteArray(), ChunkInfo.getFromProtoBuf(chunkInfo));
+ return SmallFileUtils.getGetSmallFileResponseSuccess(request, dataBuf,
+ ChunkInfo.getFromProtoBuf(chunkInfo));
} catch (StorageContainerException e) {
return ContainerUtils.logAndReturnError(LOG, e, request);
} catch (IOException ex) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index ba2b02c..5aacee9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -69,13 +69,13 @@ public final class SmallFileUtils {
* @return Response.
*/
public static ContainerCommandResponseProto getGetSmallFileResponseSuccess(
- ContainerCommandRequestProto msg, byte[] data, ChunkInfo info) {
+ ContainerCommandRequestProto msg, ByteString data, ChunkInfo info) {
Preconditions.checkNotNull(msg);
ContainerProtos.ReadChunkResponseProto.Builder readChunkresponse =
ContainerProtos.ReadChunkResponseProto.newBuilder();
readChunkresponse.setChunkData(info.getProtoBufMessage());
- readChunkresponse.setData(ByteString.copyFrom(data));
+ readChunkresponse.setData((data));
readChunkresponse.setBlockID(msg.getGetSmallFile().getBlock().getBlockID());
ContainerProtos.GetSmallFileResponseProto.Builder getSmallFile =
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org