You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2021/05/17 20:29:36 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-5222. ReplicaIndex in
Pipeline should be deserialized in the protobuf message (#2241)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 132911c HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
132911c is described below
commit 132911c504e16766ac9063cf1eaeb7f5eaa41b31
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Mon May 17 21:29:09 2021 +0100
HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
---
.../org/apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +++++++++++----
.../org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java | 11 +++++++++++
.../org/apache/hadoop/hdds/scm/pipeline/TestPipeline.java | 13 +++++++++++++
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index c440b64..33bb70a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -283,9 +283,16 @@ public final class Pipeline {
throws UnknownPipelineStateException {
Preconditions.checkNotNull(pipeline, "Pipeline is null");
- List<DatanodeDetails> nodes = new ArrayList<>();
+ Map<DatanodeDetails, Integer> nodes = new LinkedHashMap<>();
+ int index = 0;
+ int repIndexListLength = pipeline.getMemberReplicaIndexesCount();
for (DatanodeDetailsProto member : pipeline.getMembersList()) {
- nodes.add(DatanodeDetails.getFromProtoBuf(member));
+ int repIndex = 0;
+ if (index < repIndexListLength) {
+ repIndex = pipeline.getMemberReplicaIndexes(index);
+ }
+ nodes.put(DatanodeDetails.getFromProtoBuf(member), repIndex);
+ index++;
}
UUID leaderId = null;
if (pipeline.hasLeaderID128()) {
@@ -309,7 +316,8 @@ public final class Pipeline {
return new Builder().setId(PipelineID.getFromProtobuf(pipeline.getId()))
.setReplicationConfig(config)
.setState(PipelineState.fromProtobuf(pipeline.getState()))
- .setNodes(nodes)
+ .setNodes(new ArrayList<>(nodes.keySet()))
+ .setReplicaIndexes(nodes)
.setLeaderId(leaderId)
.setSuggestedLeaderId(suggestedLeaderId)
.setNodesInOrder(pipeline.getMemberOrdersList())
@@ -480,7 +488,6 @@ public final class Pipeline {
this.leaderId = leaderId1;
return this;
}
-
public Builder setNodes(List<DatanodeDetails> nodes) {
this.nodeStatus = new LinkedHashMap<>();
nodes.forEach(node -> nodeStatus.put(node, -1L));
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java
index 1212b47..259b57c 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdds.scm.pipeline;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
@@ -97,12 +99,21 @@ public final class MockPipeline {
nodes.add(MockDatanodeDetails.randomDatanodeDetails());
nodes.add(MockDatanodeDetails.randomDatanodeDetails());
+ Map<DatanodeDetails, Integer> nodeIndexes = new HashMap<>();
+
+ int index = nodes.size() - 1;
+ for (DatanodeDetails dn : nodes) {
+ nodeIndexes.put(dn, index);
+ index--;
+ }
+
return Pipeline.newBuilder()
.setState(Pipeline.PipelineState.OPEN)
.setId(PipelineID.randomId())
.setReplicationConfig(
new ECReplicationConfig(3, 2))
.setNodes(nodes)
+ .setReplicaIndexes(nodeIndexes)
.build();
}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipeline.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipeline.java
index 2cf3bf0..0b9e9fe 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipeline.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipeline.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.pipeline;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.junit.Assert;
import org.junit.Test;
@@ -68,4 +69,16 @@ public class TestPipeline {
.assertEquals(2, protobufMessage.getEcReplicationConfig().getParity());
}
+
+ @Test
+ public void testReplicaIndexesSerialisedCorrectly() throws IOException {
+ Pipeline pipeline = MockPipeline.createEcPipeline();
+ HddsProtos.Pipeline protobufMessage = pipeline.getProtobufMessage(1);
+ Pipeline reloadedPipeline = Pipeline.getFromProtobuf(protobufMessage);
+
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ Assert.assertEquals(pipeline.getReplicaIndex(dn),
+ reloadedPipeline.getReplicaIndex(dn));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org