You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2022/09/01 10:40:30 UTC
[GitHub] [ozone] lokeshj1703 commented on a diff in pull request #3701: HDDS-7155. [DiskBalancer] Create interface between SCM and DN
lokeshj1703 commented on code in PR #3701:
URL: https://github.com/apache/ozone/pull/3701#discussion_r960486276
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java:
##########
@@ -95,40 +101,54 @@ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
- }
- return statusList;
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus =
+ status.orElse(HddsProtos.DiskBalancerRunningStatus.RUNNING);
+
+ List<DatanodeDetails> finalFilterDns = filterDns;
+
+ return nodeManager.getAllNodes().stream()
Review Comment:
We are still looping through all nodes here. In case hosts is not empty we should only loop through nodes provided by the hosts input parameter.
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java:
##########
@@ -112,11 +112,11 @@ public class StateContext {
private final ConfigurationSource conf;
private final Set<InetSocketAddress> endpoints;
// Only the latest full report of each type is kept
- private final AtomicReference<GeneratedMessage> containerReports;
- private final AtomicReference<GeneratedMessage> nodeReport;
- private final AtomicReference<GeneratedMessage> pipelineReports;
- private final AtomicReference<GeneratedMessage> crlStatusReport;
- private final AtomicReference<GeneratedMessage> diskBalancerReport;
+ private final AtomicReference<Message> containerReports;
+ private final AtomicReference<Message> nodeReport;
+ private final AtomicReference<Message> pipelineReports;
+ private final AtomicReference<Message> crlStatusReport;
+ private final AtomicReference<Message> diskBalancerReport;
Review Comment:
Why do we need this change?
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java:
##########
@@ -95,40 +101,54 @@ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
- }
- return statusList;
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus =
+ status.orElse(HddsProtos.DiskBalancerRunningStatus.RUNNING);
Review Comment:
Lets not filter by default.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java:
##########
@@ -95,40 +101,54 @@ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
- }
- return statusList;
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus =
+ status.orElse(HddsProtos.DiskBalancerRunningStatus.RUNNING);
+
+ List<DatanodeDetails> finalFilterDns = filterDns;
+
+ return nodeManager.getAllNodes().stream()
+ .filter(dn -> shouldReturnDatanode(finalFilterDns, filterStatus, dn))
+ .map(dn -> {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(dn);
+ HddsProtos.DiskBalancerRunningStatus runningStatus =
+ getRunningStatus(dn);
+ HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
+ HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(dn.toProto(clientVersion))
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setRunningStatus(getRunningStatus(dn));
+ if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ builder.setDiskBalancerConf(statusMap.get(dn)
+ .getDiskBalancerConfiguration().toProtobufBuilder());
+ }
Review Comment:
Do we need to report conf for STOPPED status dns?
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java:
##########
@@ -95,40 +101,54 @@ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
- }
- return statusList;
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus =
+ status.orElse(HddsProtos.DiskBalancerRunningStatus.RUNNING);
+
+ List<DatanodeDetails> finalFilterDns = filterDns;
+
+ return nodeManager.getAllNodes().stream()
+ .filter(dn -> shouldReturnDatanode(finalFilterDns, filterStatus, dn))
+ .map(dn -> {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(dn);
+ HddsProtos.DiskBalancerRunningStatus runningStatus =
+ getRunningStatus(dn);
+ HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
+ HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(dn.toProto(clientVersion))
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setRunningStatus(getRunningStatus(dn));
+ if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ builder.setDiskBalancerConf(statusMap.get(dn)
+ .getDiskBalancerConfiguration().toProtobufBuilder());
+ }
+ return builder.build();
+ }).collect(Collectors.toList());
}
private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
- if (hosts == null || hosts.isEmpty()) {
- return isRunning(datanodeDetails);
- } else {
- return hosts.contains(datanodeDetails);
+ boolean shouldReturn = true;
+ // If dn not in specified hosts, do not return
+ if (hosts != null && !hosts.isEmpty() && !hosts.contains(datanodeDetails)) {
Review Comment:
`!hosts.isEmpty()` We do not need this check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org