You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/17 15:52:23 UTC
hadoop git commit: HDDS-3. Send NodeReport and ContainerReport when
datanodes register. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/trunk 41ae5c500 -> 59bde0992
HDDS-3. Send NodeReport and ContainerReport when datanodes register. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59bde099
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59bde099
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59bde099
Branch: refs/heads/trunk
Commit: 59bde09920fc6a1552eb31e83bbc8fc8a0f2cf22
Parents: 41ae5c5
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu May 17 08:52:00 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu May 17 08:52:00 2018 -0700
----------------------------------------------------------------------
.../states/datanode/RunningDatanodeState.java | 1 +
.../states/endpoint/RegisterEndpointTask.java | 37 ++++++++++--
.../StorageContainerDatanodeProtocol.java | 8 ++-
.../protocol/StorageContainerNodeProtocol.java | 4 +-
...rDatanodeProtocolClientSideTranslatorPB.java | 8 ++-
...rDatanodeProtocolServerSideTranslatorPB.java | 9 ++-
.../StorageContainerDatanodeProtocol.proto | 2 +
.../ozone/container/common/ScmTestMock.java | 60 +++++++++++++++++---
.../hadoop/hdds/scm/node/SCMNodeManager.java | 6 +-
.../scm/server/SCMDatanodeProtocolServer.java | 16 +++++-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 23 +++++++-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 3 +-
.../ozone/container/common/TestEndPoint.java | 32 +++++++++--
.../testutils/ReplicationNodeManagerMock.java | 5 +-
15 files changed, 186 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
index 7a8c17b..3e11d12 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
@@ -101,6 +101,7 @@ public class RunningDatanodeState implements DatanodeState {
.setConfig(conf)
.setEndpointStateMachine(endpoint)
.setDatanodeDetails(context.getParent().getDatanodeDetails())
+ .setOzoneContainer(context.getParent().getContainer())
.build();
case HEARTBEAT:
return HeartbeatEndpointTask.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index 42568e3..77a7084 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -20,12 +20,16 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,18 +49,21 @@ public final class RegisterEndpointTask implements
private final Configuration conf;
private Future<EndpointStateMachine.EndPointStates> result;
private DatanodeDetails datanodeDetails;
+ private final OzoneContainer datanodeContainerManager;
/**
* Creates a register endpoint task.
*
* @param rpcEndPoint - endpoint
* @param conf - conf
+ * @param ozoneContainer - container
*/
@VisibleForTesting
public RegisterEndpointTask(EndpointStateMachine rpcEndPoint,
- Configuration conf) {
+ Configuration conf, OzoneContainer ozoneContainer) {
this.rpcEndPoint = rpcEndPoint;
this.conf = conf;
+ this.datanodeContainerManager = ozoneContainer;
}
@@ -97,9 +104,13 @@ public final class RegisterEndpointTask implements
rpcEndPoint.lock();
try {
+ ContainerReportsRequestProto contianerReport = datanodeContainerManager
+ .getContainerReport();
+ SCMNodeReport nodeReport = datanodeContainerManager.getNodeReport();
// TODO : Add responses to the command Queue.
SCMRegisteredCmdResponseProto response = rpcEndPoint.getEndPoint()
- .register(datanodeDetails.getProtoBufMessage());
+ .register(datanodeDetails.getProtoBufMessage(), nodeReport,
+ contianerReport);
Preconditions.checkState(UUID.fromString(response.getDatanodeUUID())
.equals(datanodeDetails.getUuid()),
"Unexpected datanode ID in the response.");
@@ -139,6 +150,7 @@ public final class RegisterEndpointTask implements
private EndpointStateMachine endPointStateMachine;
private Configuration conf;
private DatanodeDetails datanodeDetails;
+ private OzoneContainer container;
/**
* Constructs the builder class.
@@ -179,6 +191,17 @@ public final class RegisterEndpointTask implements
return this;
}
+ /**
+ * Sets the ozonecontainer.
+ * @param ozoneContainer
+ * @return Builder
+ */
+ public Builder setOzoneContainer(OzoneContainer ozoneContainer) {
+ this.container = ozoneContainer;
+ return this;
+ }
+
+
public RegisterEndpointTask build() {
if (endPointStateMachine == null) {
LOG.error("No endpoint specified.");
@@ -198,8 +221,14 @@ public final class RegisterEndpointTask implements
"construct RegisterEndpoint task");
}
+ if (container == null) {
+ LOG.error("Container is not specified");
+ throw new IllegalArgumentException("Container is not specified to " +
+ "constrict RegisterEndpoint task");
+ }
+
RegisterEndpointTask task = new RegisterEndpointTask(this
- .endPointStateMachine, this.conf);
+ .endPointStateMachine, this.conf, this.container);
task.setDatanodeDetails(datanodeDetails);
return task;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index cb657276..8aa397b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -69,11 +69,13 @@ public interface StorageContainerDatanodeProtocol {
/**
* Register Datanode.
* @param datanodeDetails - Datanode Details.
- *
+ * @param nodeReport - Node Report.
+ * @param containerReportsRequestProto - Container Reports.
* @return SCM Command.
*/
- SCMRegisteredCmdResponseProto register(DatanodeDetailsProto datanodeDetails)
- throws IOException;
+ SCMRegisteredCmdResponseProto register(DatanodeDetailsProto datanodeDetails,
+ SCMNodeReport nodeReport, ContainerReportsRequestProto
+ containerReportsRequestProto) throws IOException;
/**
* Send a container report.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
index 1fc7c57..5d1d434 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -51,9 +51,11 @@ public interface StorageContainerNodeProtocol {
/**
* Register the node if the node finds that it is not registered with any SCM.
* @param datanodeDetails DatanodeDetails
+ * @param nodeReport SCMNodeReport
* @return SCMHeartbeatResponseProto
*/
- SCMCommand register(DatanodeDetailsProto datanodeDetails);
+ SCMCommand register(DatanodeDetailsProto datanodeDetails, SCMNodeReport
+ nodeReport);
/**
* Send heartbeat to indicate the datanode is alive and doing well.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
index 13162de..95d4cb0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
@@ -152,14 +152,20 @@ public class StorageContainerDatanodeProtocolClientSideTranslatorPB
* Register Datanode.
*
* @param datanodeDetailsProto - Datanode Details
+ * @param nodeReport - Node Report.
+ * @param containerReportsRequestProto - Container Reports.
* @return SCM Command.
*/
@Override
public SCMRegisteredCmdResponseProto register(
- DatanodeDetailsProto datanodeDetailsProto) throws IOException {
+ DatanodeDetailsProto datanodeDetailsProto, SCMNodeReport nodeReport,
+ ContainerReportsRequestProto containerReportsRequestProto)
+ throws IOException {
SCMRegisterRequestProto.Builder req =
SCMRegisterRequestProto.newBuilder();
req.setDatanodeDetails(datanodeDetailsProto);
+ req.setContainerReport(containerReportsRequestProto);
+ req.setNodeReport(nodeReport);
final SCMRegisteredCmdResponseProto response;
try {
response = rpcProxy.register(NULL_RPC_CONTROLLER, req.build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
index 25757aa..139f04c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
@@ -69,7 +71,12 @@ public class StorageContainerDatanodeProtocolServerSideTranslatorPB
register(RpcController controller, StorageContainerDatanodeProtocolProtos
.SCMRegisterRequestProto request) throws ServiceException {
try {
- return impl.register(request.getDatanodeDetails());
+ ContainerReportsRequestProto containerRequestProto = null;
+ SCMNodeReport scmNodeReport = null;
+ containerRequestProto = request.getContainerReport();
+ scmNodeReport = request.getNodeReport();
+ return impl.register(request.getDatanodeDetails(), scmNodeReport,
+ containerRequestProto);
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 4d5795a..f44abc9 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -146,6 +146,8 @@ message SCMStorageReport {
message SCMRegisterRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
+ required SCMNodeReport nodeReport = 2;
+ required ContainerReportsRequestProto containerReport = 3;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 703878c..948367a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -37,16 +37,13 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -62,6 +59,7 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
// Map of datanode to containers
private Map<DatanodeDetails, Map<String, ContainerInfo>> nodeContainers =
new HashMap();
+ private Map<DatanodeDetails, SCMNodeReport> nodeReports = new HashMap<>();
/**
* Returns the number of heartbeats made to this class.
*
@@ -200,9 +198,13 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
@Override
public StorageContainerDatanodeProtocolProtos
.SCMRegisteredCmdResponseProto register(
- DatanodeDetailsProto datanodeDetailsProto)
+ DatanodeDetailsProto datanodeDetailsProto, SCMNodeReport nodeReport,
+ StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto
+ containerReportsRequestProto)
throws IOException {
rpcCount.incrementAndGet();
+ sendContainerReport(containerReportsRequestProto);
+ updateNodeReport(datanodeDetailsProto, nodeReport);
sleepIfNeeded();
return StorageContainerDatanodeProtocolProtos
.SCMRegisteredCmdResponseProto
@@ -213,6 +215,50 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
}
/**
+ * Update nodeReport.
+ * @param datanodeDetailsProto
+ * @param nodeReport
+ */
+ public void updateNodeReport(DatanodeDetailsProto datanodeDetailsProto,
+ SCMNodeReport nodeReport) {
+ DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf(
+ datanodeDetailsProto);
+ SCMNodeReport.Builder datanodeReport = SCMNodeReport.newBuilder();
+
+ List<SCMStorageReport> storageReports =
+ nodeReport.getStorageReportList();
+
+ for(SCMStorageReport report : storageReports) {
+ datanodeReport.addStorageReport(report);
+ }
+
+ nodeReports.put(datanode, datanodeReport.build());
+
+ }
+
+ /**
+ * Return the number of StorageReports of a datanode.
+ * @param datanodeDetails
+ * @return count of containers of a datanode
+ */
+ public int getNodeReportsCount(DatanodeDetails datanodeDetails) {
+ return nodeReports.get(datanodeDetails).getStorageReportCount();
+ }
+
+ /**
+ * Returns the number of containers of a datanode.
+ * @param datanodeDetails
+ * @return count of storage reports of a datanode
+ */
+ public int getContainerCountsForDatanode(DatanodeDetails datanodeDetails) {
+ Map<String, ContainerInfo> cr = nodeContainers.get(datanodeDetails);
+ if(cr != null) {
+ return cr.size();
+ }
+ return 0;
+ }
+
+ /**
* Send a container report.
*
* @param reports -- Container report
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index c72e2a1..cf1d8a5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -739,11 +739,13 @@ public class SCMNodeManager
* This function generates and assigns new datanode ID
* for the datanode. This allows SCM to be run independent
* of Namenode if required.
+ * @param nodeReport NodeReport.
*
* @return SCMHeartbeatResponseProto
*/
@Override
- public SCMCommand register(DatanodeDetailsProto datanodeDetailsProto) {
+ public SCMCommand register(DatanodeDetailsProto datanodeDetailsProto,
+ SCMNodeReport nodeReport) {
String hostname = null;
String ip = null;
@@ -788,6 +790,8 @@ public class SCMNodeManager
.setErrorCode(ErrorCode.errorNodeNotPermitted)
.build();
}
+ // Updating Node Report, as registration is successful
+ updateNodeStat(datanodeDetails.getUuid(), nodeReport);
LOG.info("Data node with ID: {} Registered.",
datanodeDetails.getUuid());
RegisteredCommand.Builder builder =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index cb7a484..91ed032 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMReregisterCmdResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCmdType;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
@@ -167,11 +169,19 @@ public class SCMDatanodeProtocolServer implements
@Override
public SCMRegisteredCmdResponseProto register(
- HddsProtos.DatanodeDetailsProto datanodeDetails)
+ HddsProtos.DatanodeDetailsProto datanodeDetails, SCMNodeReport nodeReport,
+ ContainerReportsRequestProto containerReportsRequestProto)
throws IOException {
// TODO : Return the list of Nodes that forms the SCM HA.
- return getRegisteredResponse(scm.getScmNodeManager()
- .register(datanodeDetails));
+ RegisteredCommand registeredCommand = (RegisteredCommand) scm
+ .getScmNodeManager().register(datanodeDetails, nodeReport);
+ SCMCmdType type = registeredCommand.getType();
+ if (type == SCMCmdType.registeredCommand && registeredCommand.getError()
+ == SCMRegisteredCmdResponseProto.ErrorCode.success) {
+ scm.getScmContainerManager().processContainerReports(
+ containerReportsRequestProto);
+ }
+ return getRegisteredResponse(registeredCommand);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 40d8a69..ab94ef6 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -16,6 +16,10 @@
*/
package org.apache.hadoop.hdds.scm;
+import org.apache.hadoop.hdds.protocol
+ .proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -46,11 +50,28 @@ public final class TestUtils {
public static DatanodeDetails getDatanodeDetails(SCMNodeManager nodeManager,
String uuid) {
DatanodeDetails datanodeDetails = getDatanodeDetails(uuid);
- nodeManager.register(datanodeDetails.getProtoBufMessage());
+ nodeManager.register(datanodeDetails.getProtoBufMessage(), null);
return datanodeDetails;
}
/**
+ * Create Node Report object.
+ * @return SCMNodeReport
+ */
+ public static SCMNodeReport createNodeReport() {
+ SCMNodeReport.Builder nodeReport = SCMNodeReport.newBuilder();
+ for (int i = 0; i < 1; i++) {
+ SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+ nodeReport.addStorageReport(i, srb.setStorageUuid("disk")
+ .setCapacity(100)
+ .setScmUsed(10)
+ .setRemaining(90)
+ .build());
+ }
+ return nodeReport.build();
+ }
+
+ /**
* Get specified number of DatanodeDetails and registered them with node
* manager.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index f38b6f9..d8b8b5e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -373,10 +373,12 @@ public class MockNodeManager implements NodeManager {
* SCM.
*
* @param datanodeDetails DatanodeDetailsProto
+ * @param nodeReport SCMNodeReport
* @return SCMHeartbeatResponseProto
*/
@Override
- public SCMCommand register(HddsProtos.DatanodeDetailsProto datanodeDetails) {
+ public SCMCommand register(HddsProtos.DatanodeDetailsProto datanodeDetails,
+ SCMNodeReport nodeReport) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 89ce12e..117c258 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -282,7 +282,8 @@ public class TestNodeManager {
100, TimeUnit.MILLISECONDS);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- nodemanager.register(datanodeDetails.getProtoBufMessage());
+ nodemanager.register(datanodeDetails.getProtoBufMessage(),
+ TestUtils.createNodeReport());
List<SCMCommand> command = nodemanager.sendHeartbeat(
datanodeDetails.getProtoBufMessage(),
null, reportState);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 0f9125b..da39bb3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -55,12 +55,14 @@ import org.apache.hadoop.ozone.container.common.states.endpoint
.RegisterEndpointTask;
import org.apache.hadoop.ozone.container.common.states.endpoint
.VersionEndpointTask;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.mockito.Mockito.mock;
import java.io.File;
import java.net.InetSocketAddress;
@@ -75,6 +77,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils
.createEndpoint;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.mockito.Mockito.when;
/**
* Tests the endpoints.
@@ -208,11 +211,17 @@ public class TestEndPoint {
createEndpoint(
SCMTestUtils.getConf(), serverAddress, 1000)) {
SCMRegisteredCmdResponseProto responseProto = rpcEndPoint.getEndPoint()
- .register(nodeToRegister.getProtoBufMessage());
+ .register(nodeToRegister.getProtoBufMessage(),
+ TestUtils.createNodeReport(),
+ createContainerReport(10, nodeToRegister));
Assert.assertNotNull(responseProto);
Assert.assertEquals(nodeToRegister.getUuidString(),
responseProto.getDatanodeUUID());
Assert.assertNotNull(responseProto.getClusterID());
+ Assert.assertEquals(10, scmServerImpl.
+ getContainerCountsForDatanode(nodeToRegister));
+ Assert.assertEquals(1, scmServerImpl.getNodeReportsCount(
+ nodeToRegister));
}
}
@@ -223,8 +232,13 @@ public class TestEndPoint {
createEndpoint(conf,
scmAddress, rpcTimeout);
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.REGISTER);
+ OzoneContainer ozoneContainer = mock(OzoneContainer.class);
+ when(ozoneContainer.getNodeReport()).thenReturn(TestUtils
+ .createNodeReport());
+ when(ozoneContainer.getContainerReport()).thenReturn(
+ createContainerReport(10, null));
RegisterEndpointTask endpointTask =
- new RegisterEndpointTask(rpcEndPoint, conf);
+ new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer);
if (!clearDatanodeDetails) {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
endpointTask.setDatanodeDetails(datanodeDetails);
@@ -419,7 +433,8 @@ public class TestEndPoint {
createEndpoint(SCMTestUtils.getConf(),
serverAddress, 1000)) {
ContainerReportsResponseProto responseProto = rpcEndPoint
- .getEndPoint().sendContainerReport(createContainerReport(count));
+ .getEndPoint().sendContainerReport(createContainerReport(count,
+ null));
Assert.assertNotNull(responseProto);
}
Assert.assertEquals(1, scmServerImpl.getContainerReportsCount());
@@ -430,7 +445,8 @@ public class TestEndPoint {
Assert.assertEquals(expectedBytesUsed, scmServerImpl.getBytesUsed());
}
- private ContainerReportsRequestProto createContainerReport(int count) {
+ private ContainerReportsRequestProto createContainerReport(
+ int count, DatanodeDetails datanodeDetails) {
StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto.Builder
reportsBuilder = StorageContainerDatanodeProtocolProtos
.ContainerReportsRequestProto.newBuilder();
@@ -448,8 +464,12 @@ public class TestEndPoint {
reportsBuilder.addReports(report.getProtoBufMessage());
}
- reportsBuilder.setDatanodeDetails(getDatanodeDetails()
- .getProtoBufMessage());
+ if(datanodeDetails == null) {
+ reportsBuilder.setDatanodeDetails(getDatanodeDetails()
+ .getProtoBufMessage());
+ } else {
+ reportsBuilder.setDatanodeDetails(datanodeDetails.getProtoBufMessage());
+ }
reportsBuilder.setType(StorageContainerDatanodeProtocolProtos
.ContainerReportsRequestProto.reportType.fullReport);
return reportsBuilder.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59bde099/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index f2db751..b49b71b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -279,11 +279,12 @@ public class ReplicationNodeManagerMock implements NodeManager {
* Register the node if the node finds that it is not registered with any SCM.
*
* @param dd DatanodeDetailsProto
- *
+ * @param nodeReport SCMNodeReport
* @return SCMHeartbeatResponseProto
*/
@Override
- public SCMCommand register(HddsProtos.DatanodeDetailsProto dd) {
+ public SCMCommand register(HddsProtos.DatanodeDetailsProto dd,
+ SCMNodeReport nodeReport) {
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org