You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ma...@apache.org on 2022/04/02 01:58:09 UTC
[ozone] branch master updated: HDDS-5963. Implement ListBlock command (#3212)
This is an automated email from the ASF dual-hosted git repository.
markgui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2eabcc5 HDDS-5963. Implement ListBlock command (#3212)
2eabcc5 is described below
commit 2eabcc5bd5a015b5919547e1dd7f6ba388fe20be
Author: cchenax <38...@users.noreply.github.com>
AuthorDate: Sat Apr 2 09:57:34 2022 +0800
HDDS-5963. Implement ListBlock command (#3212)
Co-authored-by: cchenaxchen <cc...@tencent.com>
---
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 +
.../ContainerCommandResponseBuilders.java | 11 ++++++
.../ozone/container/ContainerTestHelper.java | 28 +++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 40 +++++++++++++++++++++-
.../container/keyvalue/impl/BlockManagerImpl.java | 10 +++---
.../container/common/impl/TestHddsDispatcher.java | 20 +++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 7 ++--
7 files changed, 108 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index ffbb3e3..55b518d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -433,6 +433,7 @@ public final class HddsUtils {
case DeleteContainer:
case ReadContainer:
case UpdateContainer:
+ case ListBlock:
return true;
default:
return false;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ContainerCommandResponseBuilders.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ContainerCommandResponseBuilders.java
index de7d083..7ef2196 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ContainerCommandResponseBuilders.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ContainerCommandResponseBuilders.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRe
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadContainerResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ListBlockResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.ozone.common.ChunkBuffer;
@@ -165,6 +166,16 @@ public final class ContainerCommandResponseBuilders {
.build();
}
+ public static ContainerCommandResponseProto getListBlockResponse(
+ ContainerCommandRequestProto msg, List<BlockData> data) {
+
+ ListBlockResponseProto.Builder builder =
+ ListBlockResponseProto.newBuilder();
+ builder.addAllBlockData(data);
+ return getSuccessResponseBuilder(msg)
+ .setListBlock(builder)
+ .build();
+ }
/**
* Returns successful getCommittedBlockLength Response.
* @param msg - Request.
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 86198cd..d90ae61 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -147,6 +147,34 @@ public final class ContainerTestHelper {
return builder.build();
}
+ public static ContainerCommandRequestProto getListBlockRequest(
+ ContainerCommandRequestProto writeChunkRequest) {
+ return ContainerCommandRequestProto.newBuilder()
+ .setContainerID(writeChunkRequest.getContainerID())
+ .setCmdType(ContainerProtos.Type.ListBlock)
+ .setDatanodeUuid(writeChunkRequest.getDatanodeUuid())
+ .setListBlock(ContainerProtos.ListBlockRequestProto.newBuilder()
+ .setCount(10).build())
+ .build();
+ }
+
+ public static ContainerCommandRequestProto getPutBlockRequest(
+ ContainerCommandRequestProto writeChunkRequest) {
+ ContainerProtos.BlockData.Builder block =
+ ContainerProtos.BlockData.newBuilder()
+ .setSize(writeChunkRequest.getWriteChunk().getChunkData().getLen())
+ .setBlockID(writeChunkRequest.getWriteChunk().getBlockID())
+ .addChunks(writeChunkRequest.getWriteChunk().getChunkData());
+ return ContainerCommandRequestProto.newBuilder()
+ .setContainerID(writeChunkRequest.getContainerID())
+ .setCmdType(ContainerProtos.Type.PutBlock)
+ .setDatanodeUuid(writeChunkRequest.getDatanodeUuid())
+ .setPutBlock(ContainerProtos.PutBlockRequestProto.newBuilder()
+ .setBlockData(block.build())
+ .build())
+ .build();
+ }
+
public static Builder newWriteChunkRequestBuilder(
Pipeline pipeline, BlockID blockID, int datalen, int seq)
throws IOException {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 73df5e4..b2e56c4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -93,6 +93,7 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Res
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getBlockDataResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getBlockLengthResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getGetSmallFileResponseSuccess;
+import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getListBlockResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getPutFileResponseSuccess;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getReadChunkResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getReadContainerResponse;
@@ -215,7 +216,7 @@ public class KeyValueHandler extends Handler {
case DeleteBlock:
return handler.handleDeleteBlock(request, kvContainer);
case ListBlock:
- return handler.handleUnsupportedOp(request);
+ return handler.handleListBlock(request, kvContainer);
case ReadChunk:
return handler.handleReadChunk(request, kvContainer, dispatcherContext);
case DeleteChunk:
@@ -547,6 +548,43 @@ public class KeyValueHandler extends Handler {
}
/**
+ * Handle List Block operation. Calls BlockManager to process the request.
+ */
+ ContainerCommandResponseProto handleListBlock(
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+
+ if (!request.hasListBlock()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Malformed list block request. trace ID: {}",
+ request.getTraceID());
+ }
+ return malformedRequest(request);
+ }
+
+ List<ContainerProtos.BlockData> returnData = new ArrayList<>();
+ try {
+ int count = request.getListBlock().getCount();
+ long startLocalId = -1;
+ if (request.getListBlock().hasStartLocalID()) {
+ startLocalId = request.getListBlock().getStartLocalID();
+ }
+ List<BlockData> responseData =
+ blockManager.listBlock(kvContainer, startLocalId, count);
+ for (int i = 0; i < responseData.size(); i++) {
+ returnData.add(responseData.get(i).getProtoBufMessage());
+ }
+ } catch (StorageContainerException ex) {
+ return ContainerUtils.logAndReturnError(LOG, ex, request);
+ } catch (IOException ex) {
+ return ContainerUtils.logAndReturnError(LOG,
+ new StorageContainerException("List blocks failed", ex, IO_EXCEPTION),
+ request);
+ }
+
+ return getListBlockResponse(request, returnData);
+ }
+
+ /**
* Handle Delete Block operation. Calls BlockManager to process the request.
*/
@Deprecated
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index ee13ffc..f0a0cb0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -317,8 +317,8 @@ public class BlockManagerImpl implements BlockManager {
public List<BlockData> listBlock(Container container, long startLocalID, int
count) throws IOException {
Preconditions.checkNotNull(container, "container cannot be null");
- Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " +
- "negative");
+ Preconditions.checkState(startLocalID >= 0 || startLocalID == -1,
+ "startLocal ID cannot be negative");
Preconditions.checkArgument(count > 0,
"Count must be a positive number.");
container.readLock();
@@ -330,11 +330,11 @@ public class BlockManagerImpl implements BlockManager {
result = new ArrayList<>();
List<? extends Table.KeyValue<String, BlockData>> range =
db.getStore().getBlockDataTable()
- .getSequentialRangeKVs(Long.toString(startLocalID), count,
+ .getSequentialRangeKVs(startLocalID == -1 ? null :
+ Long.toString(startLocalID), count,
MetadataKeyFilters.getUnprefixedKeyFilter());
for (Table.KeyValue<String, BlockData> entry : range) {
- BlockData data = new BlockData(entry.getValue().getBlockID());
- result.add(data);
+ result.add(entry.getValue());
}
return result;
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 53490bb..baceb28 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.utils.BufferUtils;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
@@ -187,6 +188,25 @@ public class TestHddsDispatcher {
response.getReadChunk().getDataBuffers().getBuffersList());
Assert.assertEquals(writeChunkRequest.getWriteChunk().getData(),
responseData);
+ // put block
+ ContainerCommandRequestProto putBlockRequest =
+ ContainerTestHelper.getPutBlockRequest(writeChunkRequest);
+ response = hddsDispatcher.dispatch(putBlockRequest, null);
+ Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
+ // send list block request
+ ContainerCommandRequestProto listBlockRequest =
+ ContainerTestHelper.getListBlockRequest(writeChunkRequest);
+ response = hddsDispatcher.dispatch(listBlockRequest, null);
+ Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
+ Assert.assertEquals(1, response.getListBlock().getBlockDataList().size());
+ for (ContainerProtos.BlockData blockData :
+ response.getListBlock().getBlockDataList()) {
+ Assert.assertEquals(writeChunkRequest.getWriteChunk().getBlockID(),
+ blockData.getBlockID());
+ Assert.assertEquals(writeChunkRequest.getWriteChunk().getChunkData()
+ .getLen(), blockData.getSize());
+ Assert.assertEquals(1, blockData.getChunksCount());
+ }
} finally {
ContainerMetrics.remove();
FileUtils.deleteDirectory(new File(testDir));
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 314a8aa..7d2d604 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -119,6 +119,7 @@ public class TestKeyValueHandler {
/**
* Test that Handler handles different command types correctly.
*/
+ @Test
public void testHandlerCommandHandling() throws Exception {
Mockito.reset(handler);
// Test Create Container Request handling
@@ -136,7 +137,7 @@ public class TestKeyValueHandler {
DispatcherContext context = new DispatcherContext.Builder().build();
KeyValueHandler
.dispatchRequest(handler, createContainerRequest, container, context);
- Mockito.verify(handler, times(1)).handleCreateContainer(
+ Mockito.verify(handler, times(0)).handleListBlock(
any(ContainerCommandRequestProto.class), any());
// Test Read Container Request handling
@@ -202,7 +203,7 @@ public class TestKeyValueHandler {
getDummyCommandRequestProto(ContainerProtos.Type.ListBlock);
KeyValueHandler
.dispatchRequest(handler, listBlockRequest, container, context);
- Mockito.verify(handler, times(2)).handleUnsupportedOp(
+ Mockito.verify(handler, times(1)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Read Chunk Request handling
@@ -229,7 +230,7 @@ public class TestKeyValueHandler {
getDummyCommandRequestProto(ContainerProtos.Type.ListChunk);
KeyValueHandler
.dispatchRequest(handler, listChunkRequest, container, context);
- Mockito.verify(handler, times(3)).handleUnsupportedOp(
+ Mockito.verify(handler, times(2)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Put Small File Request handling
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org