You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by xy...@apache.org on 2020/07/02 16:57:08 UTC
[hadoop-ozone] branch master updated: HDDS-3737. Avoid
serialization between UUID and String (#1053)
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b4798cd HDDS-3737. Avoid serialization between UUID and String (#1053)
b4798cd is described below
commit b4798cd58a1a52a117ef606956f0cc4a9efa4370
Author: runzhiwang <51...@users.noreply.github.com>
AuthorDate: Fri Jul 3 00:56:57 2020 +0800
HDDS-3737. Avoid serialization between UUID and String (#1053)
---
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 33 ++++++++++++++--------
.../scm/container/common/helpers/ExcludeList.java | 3 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 28 ++++++++++++------
.../hadoop/hdds/scm/pipeline/PipelineID.java | 23 +++++++++++----
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +-
.../container/common/helpers/DatanodeIdYaml.java | 3 +-
.../ozone/container/common/ContainerTestUtils.java | 3 +-
.../container/common/TestDatanodeStateMachine.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../common/report/TestReportPublisher.java | 3 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 2 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +--
.../interface-client/src/main/proto/hdds.proto | 17 +++++++++--
.../hadoop/hdds/scm/node/SCMNodeManager.java | 6 ++--
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +++---
.../scm/pipeline/TestPipelinePlacementPolicy.java | 2 +-
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 2 +-
.../TestContainerStateManagerIntegration.java | 4 +--
.../hadoop/ozone/recon/api/TestEndpoints.java | 10 ++++++-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +--
26 files changed, 115 insertions(+), 59 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 2fa3342..1c7779b 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -239,7 +239,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
private void addDatanodetoReply(UUID address, XceiverClientReply reply) {
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(address.toString());
+ builder.setUuid(address);
reply.addDatanode(builder.build());
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index 78d8ef1..7fa7765 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -44,7 +44,6 @@ public class DatanodeDetails extends NodeImpl implements
* DataNode's unique identifier in the cluster.
*/
private final UUID uuid;
- private final String strUuid;
private String ipAddress;
private String hostName;
@@ -61,11 +60,10 @@ public class DatanodeDetails extends NodeImpl implements
* @param ports Ports used by the DataNode
* @param certSerialId serial id from SCM issued certificate.
*/
- private DatanodeDetails(String uuid, String ipAddress, String hostName,
+ private DatanodeDetails(UUID uuid, String ipAddress, String hostName,
String networkLocation, List<Port> ports, String certSerialId) {
super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT);
- this.uuid = UUID.fromString(uuid);
- this.strUuid = uuid;
+ this.uuid = uuid;
this.ipAddress = ipAddress;
this.hostName = hostName;
this.ports = ports;
@@ -76,7 +74,6 @@ public class DatanodeDetails extends NodeImpl implements
super(datanodeDetails.getHostName(), datanodeDetails.getNetworkLocation(),
datanodeDetails.getCost());
this.uuid = datanodeDetails.uuid;
- this.strUuid = datanodeDetails.uuid.toString();
this.ipAddress = datanodeDetails.ipAddress;
this.hostName = datanodeDetails.hostName;
this.ports = datanodeDetails.ports;
@@ -98,7 +95,7 @@ public class DatanodeDetails extends NodeImpl implements
* @return UUID of DataNode
*/
public String getUuidString() {
- return strUuid;
+ return uuid.toString();
}
/**
@@ -183,7 +180,13 @@ public class DatanodeDetails extends NodeImpl implements
public static DatanodeDetails getFromProtoBuf(
HddsProtos.DatanodeDetailsProto datanodeDetailsProto) {
DatanodeDetails.Builder builder = newBuilder();
- builder.setUuid(datanodeDetailsProto.getUuid());
+ if (datanodeDetailsProto.hasUuid128()) {
+ HddsProtos.UUID uuid = datanodeDetailsProto.getUuid128();
+ builder.setUuid(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()));
+ } else if (datanodeDetailsProto.hasUuid()) {
+ builder.setUuid(UUID.fromString(datanodeDetailsProto.getUuid()));
+ }
+
if (datanodeDetailsProto.hasIpAddress()) {
builder.setIpAddress(datanodeDetailsProto.getIpAddress());
}
@@ -211,9 +214,17 @@ public class DatanodeDetails extends NodeImpl implements
* @return HddsProtos.DatanodeDetailsProto
*/
public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
+ HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder()
+ .setMostSigBits(uuid.getMostSignificantBits())
+ .setLeastSigBits(uuid.getLeastSignificantBits())
+ .build();
+
HddsProtos.DatanodeDetailsProto.Builder builder =
HddsProtos.DatanodeDetailsProto.newBuilder()
- .setUuid(getUuidString());
+ .setUuid128(uuid128);
+
+ builder.setUuid(getUuidString());
+
if (ipAddress != null) {
builder.setIpAddress(ipAddress);
}
@@ -241,7 +252,7 @@ public class DatanodeDetails extends NodeImpl implements
@Override
public String toString() {
- return strUuid + "{" +
+ return uuid.toString() + "{" +
"ip: " +
ipAddress +
", host: " +
@@ -281,7 +292,7 @@ public class DatanodeDetails extends NodeImpl implements
* Builder class for building DatanodeDetails.
*/
public static final class Builder {
- private String id;
+ private UUID id;
private String ipAddress;
private String hostName;
private String networkName;
@@ -303,7 +314,7 @@ public class DatanodeDetails extends NodeImpl implements
* @param uuid DatanodeUuid
* @return DatanodeDetails.Builder
*/
- public Builder setUuid(String uuid) {
+ public Builder setUuid(UUID uuid) {
this.id = uuid;
return this;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
index f6b111f..803aa03 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
+import java.util.UUID;
/**
* This class contains set of dns and containers which ozone client provides
@@ -94,7 +95,7 @@ public class ExcludeList {
});
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
excludeListProto.getDatanodesList().forEach(dn -> {
- builder.setUuid(dn);
+ builder.setUuid(UUID.fromString(dn));
excludeList.addDatanode(builder.build());
});
excludeListProto.getPipelineIdsList().forEach(pipelineID -> {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 4607ce5..d5c1024 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -59,7 +59,6 @@ public final class Pipeline {
private ThreadLocal<List<DatanodeDetails>> nodesInOrder = new ThreadLocal<>();
// Current reported Leader for the pipeline
private UUID leaderId;
- private String strLeaderID = "";
// Timestamp for pipeline upon creation
private Instant creationTimestamp;
@@ -147,9 +146,6 @@ public final class Pipeline {
*/
void setLeaderId(UUID leaderId) {
this.leaderId = leaderId;
- if (leaderId != null) {
- this.strLeaderID = leaderId.toString();
- }
}
/**
@@ -270,9 +266,18 @@ public final class Pipeline {
.setType(type)
.setFactor(factor)
.setState(PipelineState.getProtobuf(state))
- .setLeaderID(strLeaderID)
+ .setLeaderID(leaderId != null ? leaderId.toString() : "")
.setCreationTimeStamp(creationTimestamp.toEpochMilli())
.addAllMembers(members);
+
+ if (leaderId != null) {
+ HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder()
+ .setMostSigBits(leaderId.getMostSignificantBits())
+ .setLeastSigBits(leaderId.getLeastSignificantBits())
+ .build();
+ builder.setLeaderID128(uuid128);
+ }
+
// To save the message size on wire, only transfer the node order based on
// network topology
List<DatanodeDetails> nodes = nodesInOrder.get();
@@ -301,14 +306,21 @@ public final class Pipeline {
for (DatanodeDetailsProto member : pipeline.getMembersList()) {
nodes.add(DatanodeDetails.getFromProtoBuf(member));
}
+ UUID leaderId = null;
+ if (pipeline.hasLeaderID128()) {
+ HddsProtos.UUID uuid = pipeline.getLeaderID128();
+ leaderId = new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits());
+ } else if (pipeline.hasLeaderID() &&
+ StringUtils.isNotEmpty(pipeline.getLeaderID())) {
+ leaderId = UUID.fromString(pipeline.getLeaderID());
+ }
return new Builder().setId(PipelineID.getFromProtobuf(pipeline.getId()))
.setFactor(pipeline.getFactor())
.setType(pipeline.getType())
.setState(PipelineState.fromProtobuf(pipeline.getState()))
- .setLeaderId(StringUtils.isNotEmpty(pipeline.getLeaderID()) ?
- UUID.fromString(pipeline.getLeaderID()) : null)
.setNodes(nodes)
+ .setLeaderId(leaderId)
.setNodesInOrder(pipeline.getMemberOrdersList())
.setCreateTimestamp(pipeline.getCreationTimeStamp())
.build();
@@ -353,7 +365,7 @@ public final class Pipeline {
b.append(", Type:").append(getType());
b.append(", Factor:").append(getFactor());
b.append(", State:").append(getPipelineState());
- b.append(", leaderId:").append(strLeaderID);
+ b.append(", leaderId:").append(leaderId != null ? leaderId.toString() : "");
b.append(", CreationTimestamp").append(getCreationTimestamp());
b.append("]");
return b.toString();
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
index 742d807..ed7d215 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
@@ -28,11 +28,9 @@ import java.util.UUID;
public final class PipelineID {
private UUID id;
- private String strID;
private PipelineID(UUID id) {
this.id = id;
- this.strID = id.toString();
}
public static PipelineID randomId() {
@@ -48,16 +46,31 @@ public final class PipelineID {
}
public HddsProtos.PipelineID getProtobuf() {
- return HddsProtos.PipelineID.newBuilder().setId(strID).build();
+ HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder()
+ .setMostSigBits(id.getMostSignificantBits())
+ .setLeastSigBits(id.getLeastSignificantBits())
+ .build();
+
+ return HddsProtos.PipelineID.newBuilder().setId(id.toString())
+ .setUuid128(uuid128).build();
}
public static PipelineID getFromProtobuf(HddsProtos.PipelineID protos) {
- return new PipelineID(UUID.fromString(protos.getId()));
+ if (protos.hasUuid128()) {
+ HddsProtos.UUID uuid = protos.getUuid128();
+ return new PipelineID(
+ new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()));
+ } else if (protos.hasId()) {
+ return new PipelineID(UUID.fromString(protos.getId()));
+ } else {
+ throw new IllegalArgumentException(
+ "Pipeline does not has uuid128 in proto");
+ }
}
@Override
public String toString() {
- return "PipelineID=" + strID;
+ return "PipelineID=" + id.toString();
}
@Override
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java
index 0cf3b25..5792719 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java
@@ -94,7 +94,7 @@ public final class MockDatanodeDetails {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, port);
return DatanodeDetails.newBuilder()
- .setUuid(uuid)
+ .setUuid(UUID.fromString(uuid))
.setHostName(hostname)
.setIpAddress(ipAddress)
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 615588a..7e896e7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -394,8 +394,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
} else {
// There is no datanode.id file, this might be the first time datanode
// is started.
- String datanodeUuid = UUID.randomUUID().toString();
- return DatanodeDetails.newBuilder().setUuid(datanodeUuid).build();
+ return DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build();
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
index d3efa98..44a12c2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
@@ -25,6 +25,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.UUID;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -78,7 +79,7 @@ public final class DatanodeIdYaml {
}
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(datanodeDetailsYaml.getUuid())
+ builder.setUuid(UUID.fromString(datanodeDetailsYaml.getUuid()))
.setIpAddress(datanodeDetailsYaml.getIpAddress())
.setHostName(datanodeDetailsYaml.getHostName())
.setCertSerialId(datanodeDetailsYaml.getCertSerialId());
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
index aabde54..15cd4d0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
@@ -98,7 +98,6 @@ public final class ContainerTestUtils {
random.nextInt(256) + "." + random.nextInt(256) + "." + random
.nextInt(256) + "." + random.nextInt(256);
- String uuid = UUID.randomUUID().toString();
DatanodeDetails.Port containerPort =
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort =
@@ -106,7 +105,7 @@ public final class ContainerTestUtils {
DatanodeDetails.Port restPort =
DatanodeDetails.newPort(DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(uuid)
+ builder.setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress(ipAddress)
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 65ae6ce..35858fd 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -427,7 +427,7 @@ public class TestDatanodeStateMachine {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
return DatanodeDetails.newBuilder()
- .setUuid(UUID.randomUUID().toString())
+ .setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index b60ee4c..e6998fe 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -305,7 +305,7 @@ public class TestHddsDispatcher {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(UUID.randomUUID().toString())
+ builder.setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index 166aadf..aa1337f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -166,7 +166,6 @@ public class TestReportPublisher {
* @return DatanodeDetails
*/
private static DatanodeDetails getDatanodeDetails() {
- String uuid = UUID.randomUUID().toString();
Random random = new Random();
String ipAddress =
random.nextInt(256) + "." + random.nextInt(256) + "." + random
@@ -179,7 +178,7 @@ public class TestReportPublisher {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(uuid)
+ builder.setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress(ipAddress)
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
index 5182c1c..ae9e428 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
@@ -233,7 +233,7 @@ public class TestCloseContainerCommandHandler {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(UUID.randomUUID().toString())
+ builder.setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress(ipAddress)
.addPort(containerPort)
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
index 1c66d63..9b238a1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
@@ -268,7 +268,7 @@ public class TestHeartbeatEndpointTask {
StateContext context,
StorageContainerDatanodeProtocolClientSideTranslatorPB proxy) {
DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder()
- .setUuid(UUID.randomUUID().toString())
+ .setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.build();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
index a9b28c8..d18d145 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
@@ -135,7 +135,7 @@ public class TestKeyValueHandlerWithUnhealthyContainer {
private KeyValueHandler getDummyHandler() throws IOException {
OzoneConfiguration conf = new OzoneConfiguration();
DatanodeDetails dnDetails = DatanodeDetails.newBuilder()
- .setUuid(DATANODE_UUID)
+ .setUuid(UUID.fromString(DATANODE_UUID))
.setHostName("dummyHost")
.setIpAddress("1.2.3.4")
.build();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 77dcb26..1056a0d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -260,8 +260,6 @@ public class TestOzoneContainer {
random.nextInt(256) + "." + random.nextInt(256) + "." + random
.nextInt(256) + "." + random.nextInt(256);
- String uuid = UUID.randomUUID().toString();
- String hostName = uuid;
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
@@ -269,7 +267,7 @@ public class TestOzoneContainer {
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(uuid)
+ builder.setUuid(UUID.randomUUID())
.setHostName("localhost")
.setIpAddress(ipAddress)
.addPort(containerPort)
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index c78175c..23cc9cb 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -28,8 +28,14 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.hdds;
+message UUID {
+ required int64 mostSigBits = 1;
+ required int64 leastSigBits = 2;
+}
+
message DatanodeDetailsProto {
- required string uuid = 1; // UUID assigned to the Datanode.
+ // deprecated, please use uuid128 instead
+ optional string uuid = 1; // UUID assigned to the Datanode.
required string ipAddress = 2; // IP address
required string hostName = 3; // hostname
repeated Port ports = 4;
@@ -37,6 +43,8 @@ message DatanodeDetailsProto {
// network name, can be Ip address or host name, depends
optional string networkName = 6;
optional string networkLocation = 7; // Network topology location
+ // TODO(runzhiwang): when uuid is gone, specify 1 as the index of uuid128 and mark as required
+ optional UUID uuid128 = 100; // UUID with 128 bits assigned to the Datanode.
}
/**
@@ -56,7 +64,10 @@ message Port {
}
message PipelineID {
- required string id = 1;
+ // deprecated, please use uuid128 instead
+ optional string id = 1;
+ // TODO(runzhiwang): when id is gone, specify 1 as the index of uuid128 and mark as required
+ optional UUID uuid128 = 100;
}
enum PipelineState {
@@ -76,6 +87,8 @@ message Pipeline {
optional string leaderID = 6;
repeated uint32 memberOrders = 7;
optional uint64 creationTimeStamp = 8;
+ // TODO(runzhiwang): when leaderID is gone, specify 6 as the index of leaderID128
+ optional UUID leaderID128 = 100;
}
message KeyValue {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 3bcd2d3..5a3851a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -615,7 +615,8 @@ public class SCMNodeManager implements NodeManager {
LOG.warn("uuid is null");
return null;
}
- DatanodeDetails temp = DatanodeDetails.newBuilder().setUuid(uuid).build();
+ DatanodeDetails temp = DatanodeDetails.newBuilder()
+ .setUuid(UUID.fromString(uuid)).build();
try {
return nodeStateManager.getNode(temp);
} catch (NodeNotFoundException e) {
@@ -645,7 +646,8 @@ public class SCMNodeManager implements NodeManager {
}
for (String uuid : uuids) {
- DatanodeDetails temp = DatanodeDetails.newBuilder().setUuid(uuid).build();
+ DatanodeDetails temp = DatanodeDetails.newBuilder()
+ .setUuid(UUID.fromString(uuid)).build();
try {
results.add(nodeStateManager.getNode(temp));
} catch (NodeNotFoundException e) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index 9843b74..75ad662 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -102,13 +102,13 @@ public class TestDeletedBlockLog {
private void setupContainerManager() throws IOException {
dnList.add(
- DatanodeDetails.newBuilder().setUuid(UUID.randomUUID().toString())
+ DatanodeDetails.newBuilder().setUuid(UUID.randomUUID())
.build());
dnList.add(
- DatanodeDetails.newBuilder().setUuid(UUID.randomUUID().toString())
+ DatanodeDetails.newBuilder().setUuid(UUID.randomUUID())
.build());
dnList.add(
- DatanodeDetails.newBuilder().setUuid(UUID.randomUUID().toString())
+ DatanodeDetails.newBuilder().setUuid(UUID.randomUUID())
.build());
final ContainerInfo container =
@@ -251,7 +251,7 @@ public class TestDeletedBlockLog {
blocks = getTransactions(50);
Assert.assertEquals(30, blocks.size());
commitTransactions(blocks, dnList.get(1), dnList.get(2),
- DatanodeDetails.newBuilder().setUuid(UUID.randomUUID().toString())
+ DatanodeDetails.newBuilder().setUuid(UUID.randomUUID())
.build());
blocks = getTransactions(50);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index b153487..8d6a28c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -310,7 +310,7 @@ public class TestPipelinePlacementPolicy {
private DatanodeDetails overwriteLocationInNode(
DatanodeDetails datanode, Node node) {
DatanodeDetails result = DatanodeDetails.newBuilder()
- .setUuid(datanode.getUuidString())
+ .setUuid(datanode.getUuid())
.setHostName(datanode.getHostName())
.setIpAddress(datanode.getIpAddress())
.addPort(datanode.getPort(DatanodeDetails.Port.Name.STANDALONE))
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
index cebedf2..3625e34 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
@@ -69,7 +69,7 @@ public class TestSCMNodeMetrics {
registeredDatanode = DatanodeDetails.newBuilder()
.setHostName("localhost")
.setIpAddress("127.0.0.1")
- .setUuid(UUID.randomUUID().toString())
+ .setUuid(UUID.randomUUID())
.build();
nodeManager.register(registeredDatanode, createNodeReport(),
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
index b779eed..3842818 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
@@ -412,10 +412,10 @@ public class TestContainerStateManagerIntegration {
public void testReplicaMap() throws Exception {
DatanodeDetails dn1 = DatanodeDetails.newBuilder().setHostName("host1")
.setIpAddress("1.1.1.1")
- .setUuid(UUID.randomUUID().toString()).build();
+ .setUuid(UUID.randomUUID()).build();
DatanodeDetails dn2 = DatanodeDetails.newBuilder().setHostName("host2")
.setIpAddress("2.2.2.2")
- .setUuid(UUID.randomUUID().toString()).build();
+ .setUuid(UUID.randomUUID()).build();
// Test 1: no replica's exist
ContainerID containerID = ContainerID.valueof(RandomUtils.nextLong());
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index 40d3ba6..a592119 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.recon.api;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID;
@@ -197,9 +198,16 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
.build())
.build();
+ UUID pipelineUuid = UUID.fromString(pipelineId);
+ HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder()
+ .setMostSigBits(pipelineUuid.getMostSignificantBits())
+ .setLeastSigBits(pipelineUuid.getLeastSignificantBits())
+ .build();
+
PipelineReport pipelineReport = PipelineReport.newBuilder()
.setPipelineID(
- PipelineID.newBuilder().setId(pipelineId).build())
+ PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128)
+ .build())
.setIsLeader(true)
.build();
PipelineReportsProto pipelineReportsProto =
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java
index 5d82d5f..8f65755 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java
@@ -182,7 +182,7 @@ public class LeaderAppendLogEntryGenerator extends BaseAppendLogGenerator
List<DatanodeDetails> datanodes = new ArrayList<>();
datanodes.add(DatanodeDetails.newBuilder()
- .setUuid(serverId)
+ .setUuid(UUID.fromString(serverId))
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.addPort(DatanodeDetails.newPort(Name.RATIS, 9858))
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
index 63e4c6e..c8872de 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
@@ -142,7 +142,7 @@ public class BenchMarkContainerStateMap {
Preconditions.checkArgument(numNodes >= 1);
final List<DatanodeDetails> ids = new ArrayList<>(numNodes);
for (int i = 0; i < numNodes; i++) {
- ids.add(GenesisUtil.createDatanodeDetails(UUID.randomUUID().toString()));
+ ids.add(GenesisUtil.createDatanodeDetails(UUID.randomUUID()));
}
return createPipeline(containerName, ids);
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
index 156b6a2..bc2a1ea 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
@@ -99,7 +99,7 @@ public final class GenesisUtil {
return builder.build();
}
- public static DatanodeDetails createDatanodeDetails(String uuid) {
+ public static DatanodeDetails createDatanodeDetails(UUID uuid) {
String ipAddress =
RANDOM.nextInt(256) + "." + RANDOM.nextInt(256) + "." + RANDOM
.nextInt(256) + "." + RANDOM.nextInt(256);
@@ -157,7 +157,7 @@ public final class GenesisUtil {
List<DatanodeDetails> nodes = new ArrayList<>();
for (int i = 0; i < factor.getNumber(); i++) {
nodes
- .add(GenesisUtil.createDatanodeDetails(UUID.randomUUID().toString()));
+ .add(GenesisUtil.createDatanodeDetails(UUID.randomUUID()));
}
for (int i = 0; i < numPipelines; i++) {
Pipeline pipeline =
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org