You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2024/02/01 16:30:49 UTC
(ozone) branch HDDS-5713 updated (816ab15c92 -> 8cc0f2de51)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 816ab15c92 HDDS-8844. Internal move logic for DiskBalancer (#4887)
omit ccc5b5182d HDDS-8182. Add volume and container choosing policy (#4408)
omit 12c3a87252 HDDS-7383. Basic framework of DiskBalancerService (#3874)
omit 43db23b367 HDDS-7234. Add a common option for DiskBalancer commands (#3762)
omit d865981dee HDDS-7205. DiskBalancer CLI (#3739)
omit 889f86fc1e HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
omit 514b16daf0 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
add ce4903de89 HDDS-9288. Remove flaky tag from TestSnapshotDeletingService.testMultipleSnapshotKeyReclaim (#6140)
add f8d54b3875 HDDS-10208. Simplify tests using assertThrows in hadoop-hdds (#6143)
add 49df6464ae HDDS-10173. findbugs: Random object used only once in TestBlockOutputStreamCorrectness. (#6131)
new e535aeadb9 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
new b4fa3514ff HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
new 4e20553120 HDDS-7205. DiskBalancer CLI (#3739)
new 80ded7c419 HDDS-7234. Add a common option for DiskBalancer commands (#3762)
new a1b69e1da9 HDDS-7383. Basic framework of DiskBalancerService (#3874)
new 9b43e3ef31 HDDS-8182. Add volume and container choosing policy (#4408)
new 8cc0f2de51 HDDS-8844. Internal move logic for DiskBalancer (#4887)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (816ab15c92)
\
N -- N -- N refs/heads/HDDS-5713 (8cc0f2de51)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hdds/scm/client/TestHddsClientUtils.java | 16 ++-----
.../storage/TestBlockOutputStreamCorrectness.java | 49 +++++++++-----------
.../TestSchemaOneBackwardsCompatibility.java | 10 +----
.../common/impl/TestContainerPersistence.java | 26 ++++-------
.../container/keyvalue/TestKeyValueContainer.java | 37 ++++++++-------
.../keyvalue/impl/TestFilePerBlockStrategy.java | 12 ++---
.../erasurecode/rawcoder/TestRawCoderBase.java | 35 ++++-----------
.../hadoop/hdds/conf/TestHddsConfServlet.java | 10 ++---
.../certificate/authority/TestDefaultCAServer.java | 28 +++++-------
.../utils/TestCertificateSignRequest.java | 23 ++++------
.../certificate/utils/TestRootCertificate.java | 37 ++++++---------
.../utils/db/TestRDBStoreByteArrayIterator.java | 11 ++---
.../utils/db/TestRDBStoreCodecBufferIterator.java | 10 ++---
.../TestSCMContainerPlacementRackAware.java | 37 +++++++--------
.../TestSCMContainerPlacementRackScatter.java | 15 +++----
.../replication/TestECOverReplicationHandler.java | 11 ++---
.../TestRatisOverReplicationHandler.java | 12 ++---
.../hdds/scm/ha/TestReplicationAnnotation.java | 15 +++----
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 10 ++---
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 52 +++++++---------------
.../scm/pipeline/TestPipelinePlacementPolicy.java | 31 +++++--------
.../scm/pipeline/TestPipelineStateManagerImpl.java | 26 +++++------
.../scm/pipeline/TestRatisPipelineProvider.java | 23 ++++------
.../scm/security/TestRootCARotationManager.java | 30 ++++---------
.../cli/datanode/TestDecommissionSubCommand.java | 9 +---
.../cli/datanode/TestMaintenanceSubCommand.java | 9 +---
.../cli/datanode/TestRecommissionSubCommand.java | 9 +---
.../ozone/om/TestSnapshotDeletingService.java | 2 -
28 files changed, 206 insertions(+), 389 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/07: HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e535aeadb9b5589fa326b8f211012dd43ee6ffe5
Author: Symious <yi...@foxmail.com>
AuthorDate: Tue Aug 16 10:40:22 2022 +0800
HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 41 +++++
.../protocol/StorageContainerLocationProtocol.java | 30 ++++
.../scm/storage/DiskBalancerConfiguration.java | 165 +++++++++++++++++++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 55 +++++++
.../src/main/proto/ScmAdminProtocol.proto | 21 +++
.../interface-client/src/main/proto/hdds.proto | 13 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 176 +++++++++++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 50 ++++++
.../org/apache/hadoop/hdds/scm/node/NodeUtils.java | 150 ++++++++++++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 33 +++-
.../hdds/scm/server/SCMClientProtocolServer.java | 50 ++++++
.../hdds/scm/server/StorageContainerManager.java | 8 +
.../hdds/scm/node/TestDiskBalancerManager.java | 99 ++++++++++++
.../hdds/scm/cli/ContainerOperationClient.java | 35 ++++
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 90 +++++++++++
16 files changed, 1017 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 402398e36c..8f09b43424 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -452,4 +452,45 @@ public interface ScmClient extends Closeable {
DecommissionScmResponseProto decommissionScm(
String scmId) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param count top datanodes that need balancing
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param hosts If hosts is not null, return status of hosts; If hosts is
+ * null, return status of all datanodes in balancing.
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index e8bddb42cf..1d7e83b247 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -474,4 +474,34 @@ public interface StorageContainerLocationProtocol extends Closeable {
DecommissionScmResponseProto decommissionScm(
String scmId) throws IOException;
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
new file mode 100644
index 0000000000..54709e4c17
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import jakarta.annotation.Nonnull;
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class contains configuration values for the DiskBalancer.
+ */
+@ConfigGroup(prefix = "hdds.datanode.disk.balancer")
+public final class DiskBalancerConfiguration {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerConfiguration.class);
+
+ @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "Threshold is a percentage in the range of 0 to 100. A " +
+ "datanode is considered balanced if for each volume, the " +
+ "utilization of the volume(used space to capacity ratio) differs" +
+ " from the utilization of the datanode(used space to capacity ratio" +
+ " of the entire datanode) no more than the threshold.")
+ private double threshold = 10d;
+
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "The max balance speed.")
+ private double diskBandwidth = 10;
+
+ @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
+ description = "The max parallel balance thread count.")
+ private int parallelThread = 5;
+
+ /**
+ * Gets the threshold value for DiskBalancer.
+ *
+ * @return percentage value in the range 0 to 100
+ */
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public double getThresholdAsRatio() {
+ return threshold / 100;
+ }
+
+ /**
+ * Sets the threshold value for Disk Balancer.
+ *
+ * @param threshold a percentage value in the range 0 to 100
+ */
+ public void setThreshold(double threshold) {
+ if (threshold < 0d || threshold >= 100d) {
+ throw new IllegalArgumentException(
+ "Threshold must be a percentage(double) in the range 0 to 100.");
+ }
+ this.threshold = threshold;
+ }
+
+ /**
+ * Gets the disk bandwidth value for Disk Balancer.
+ *
+ * @return max disk bandwidth per second
+ */
+ public double getDiskBandwidth() {
+ return diskBandwidth;
+ }
+
+ /**
+ * Sets the disk bandwidth value for Disk Balancer.
+ *
+ * @param diskBandwidth the bandwidth to control balance speed
+ */
+ public void setDiskBandwidth(double diskBandwidth) {
+ if (diskBandwidth <= 0d) {
+ throw new IllegalArgumentException(
+ "diskBandwidth must be a value larger than 0.");
+ }
+ this.diskBandwidth = diskBandwidth;
+ }
+
+ /**
+ * Gets the parallel thread for Disk Balancer.
+ *
+ * @return parallel thread
+ */
+ public int getParallelThread() {
+ return parallelThread;
+ }
+
+ /**
+ * Sets the parallel thread for Disk Balancer.
+ *
+ * @param parallelThread the parallel thread count
+ */
+ public void setParallelThread(int parallelThread) {
+ if (parallelThread <= 0) {
+ throw new IllegalArgumentException(
+ "parallelThread must be a value larger than 0.");
+ }
+ this.parallelThread = parallelThread;
+ }
+ @Override
+ public String toString() {
+ return String.format("Disk Balancer Configuration values:%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n",
+ "Key", "Value",
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Parallel Thread", parallelThread);
+ }
+
+ public HddsProtos.DiskBalancerConfigurationProto.Builder toProtobufBuilder() {
+ HddsProtos.DiskBalancerConfigurationProto.Builder builder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+
+ builder.setThreshold(threshold)
+ .setDiskBandwidth(diskBandwidth)
+ .setParallelThread(parallelThread);
+ return builder;
+ }
+
+ static DiskBalancerConfiguration fromProtobuf(
+ @Nonnull HddsProtos.DiskBalancerConfigurationProto proto,
+ @Nonnull OzoneConfiguration ozoneConfiguration) {
+ DiskBalancerConfiguration config =
+ ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ if (proto.hasThreshold()) {
+ config.setThreshold(proto.getThreshold());
+ }
+ if (proto.hasDiskBandwidth()) {
+ config.setDiskBandwidth(proto.getDiskBandwidth());
+ }
+ if (proto.hasParallelThread()) {
+ config.setParallelThread(proto.getParallelThread());
+ }
+ return config;
+ }
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 24feb69389..d4817ea592 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@ -52,5 +52,6 @@ public enum ConfigTag {
TLS,
TOKEN,
UPGRADE,
- X509
+ X509,
+ DISKBALANCER
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 84a0fa4886..62d80e3a65 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -47,6 +47,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1112,6 +1115,58 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
return response.getContainerCount();
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto request =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.report)
+ .setCount(count)
+ .build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.status);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ }
+
@Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 6adca817ed..649d61af9f 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -83,6 +83,7 @@ message ScmContainerLocationRequest {
optional DecommissionScmRequestProto decommissionScmRequest = 44;
optional SingleNodeQueryRequestProto singleNodeQueryRequest = 45;
optional GetContainersOnDecomNodeRequestProto getContainersOnDecomNodeRequest = 46;
+ optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 47;
}
message ScmContainerLocationResponse {
@@ -137,6 +138,7 @@ message ScmContainerLocationResponse {
optional DecommissionScmResponseProto decommissionScmResponse = 44;
optional SingleNodeQueryResponseProto singleNodeQueryResponse = 45;
optional GetContainersOnDecomNodeResponseProto getContainersOnDecomNodeResponse = 46;
+ optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 47;
enum Status {
OK = 1;
@@ -190,6 +192,7 @@ enum Type {
DecommissionScm = 40;
SingleNodeQuery = 41;
GetContainersOnDecomNode = 42;
+ DatanodeDiskBalancerInfo = 43;
}
/**
@@ -354,6 +357,24 @@ message DatanodeUsageInfoResponseProto {
repeated DatanodeUsageInfoProto info = 1;
}
+enum DatanodeDiskBalancerInfoType{
+ report = 1;
+ status = 2;
+}
+
+/*
+ Datanode disk balancer status request message.
+*/
+message DatanodeDiskBalancerInfoRequestProto {
+ required DatanodeDiskBalancerInfoType infoType = 1;
+ optional uint32 count = 2;
+ repeated string hosts = 3;
+}
+
+message DatanodeDiskBalancerInfoResponseProto {
+ repeated DatanodeDiskBalancerInfoProto info = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 3f346300b3..424686333a 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -497,3 +497,16 @@ message CompactionLogEntryProto {
repeated CompactionFileInfoProto outputFileIntoList = 4;
optional string compactionReason = 5;
}
+
+message DiskBalancerConfigurationProto {
+ optional double threshold = 1;
+ optional double diskBandwidth = 2;
+ optional int32 parallelThread = 3;
+}
+
+message DatanodeDiskBalancerInfoProto {
+ required DatanodeDetailsProto node = 1;
+ required double currentVolumeDensitySum = 2;
+ optional bool diskBalancerRunning = 3;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 4;
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
new file mode 100644
index 0000000000..0b16c1aa95
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+
+/**
+ * Maintains information about the DiskBalancer on SCM side.
+ */
+public class DiskBalancerManager {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private final EventPublisher scmNodeEventPublisher;
+ private final SCMContext scmContext;
+ private final NodeManager nodeManager;
+ private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private boolean useHostnames;
+
+ /**
+ * Constructs DiskBalancer Manager.
+ */
+ public DiskBalancerManager(OzoneConfiguration conf,
+ EventPublisher eventPublisher,
+ SCMContext scmContext,
+ NodeManager nodeManager) {
+ this.scmNodeEventPublisher = eventPublisher;
+ this.scmContext = scmContext;
+ this.nodeManager = nodeManager;
+ this.useHostnames = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
+ this.statusMap = new ConcurrentHashMap<>();
+ }
+
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportList =
+ new ArrayList<>();
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ reportList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+
+ reportList.sort((t1, t2) -> Double.compare(t2.getCurrentVolumeDensitySum(),
+ t1.getCurrentVolumeDensitySum()));
+ return reportList.stream().limit(count).collect(Collectors.toList());
+ }
+
+ /**
+ * If hosts is not null, return status of hosts;
+ * If hosts is null, return status of all datanodes in balancing.
+ */
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
+ new ArrayList<>();
+ List<DatanodeDetails> filterDns = null;
+ if (hosts.isPresent() && !hosts.get().isEmpty()) {
+ filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ }
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ if (shouldReturnDatanode(filterDns, datanodeDetails)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setDiskBalancerRunning(isRunning(datanodeDetails))
+ .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
+ DiskBalancerStatus.DUMMY_STATUS)
+ .getDiskBalancerConfiguration().toProtobufBuilder())
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+ }
+ return statusList;
+ }
+
+ private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ DatanodeDetails datanodeDetails) {
+ if (hosts == null || hosts.isEmpty()) {
+ return isRunning(datanodeDetails);
+ } else {
+ return hosts.contains(datanodeDetails);
+ }
+ }
+
+ /**
+ * Get volume density for a specific DatanodeDetails node.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @return DiskBalancer report.
+ */
+ private double getVolumeDataDensitySumForDatanodeDetails(
+ DatanodeDetails datanodeDetails) {
+ Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo);
+
+ DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
+
+ double totalCapacity = 0d, totalUsed = 0d;
+ for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
+ datanodeInfo.getStorageReports()) {
+ totalCapacity += reportProto.getCapacity();
+ totalUsed += reportProto.getScmUsed();
+ }
+
+ Preconditions.checkArgument(totalCapacity != 0);
+ double idealUsage = totalUsed / totalCapacity;
+
+ double volumeDensitySum = datanodeInfo.getStorageReports().stream()
+ .map(report ->
+ Math.abs((double)report.getScmUsed() / report.getCapacity()
+ - idealUsage))
+ .mapToDouble(Double::valueOf).sum();
+
+ return volumeDensitySum;
+ }
+
+ private boolean isRunning(DatanodeDetails datanodeDetails) {
+ return statusMap
+ .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
+ .isRunning();
+ }
+
+ @VisibleForTesting
+ public void addRunningDatanode(DatanodeDetails datanodeDetails) {
+ statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
+ new DiskBalancerConfiguration()));
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
new file mode 100644
index 0000000000..a064a6ff9f
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Maintains DiskBalancerConfiguration and isRunning.
+ */
+public class DiskBalancerStatus {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerStatus.class);
+
+ private boolean isRunning;
+ private DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public static final DiskBalancerStatus DUMMY_STATUS =
+ new DiskBalancerStatus(false, new DiskBalancerConfiguration());
+
+ public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
+ this.isRunning = isRunning;
+ this.diskBalancerConfiguration = conf;
+ }
+
+ public boolean isRunning() {
+ return isRunning;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
new file mode 100644
index 0000000000..6df74d3746
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Util class for Node operations.
+ */
+public final class NodeUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NodeUtils.class);
+
+ private NodeUtils() {
+ }
+
+ public static List<DatanodeDetails> mapHostnamesToDatanodes(
+ NodeManager nodeManager, List<String> hosts, boolean useHostnames)
+ throws InvalidHostStringException {
+ List<DatanodeDetails> results = new LinkedList<>();
+ for (String hostString : hosts) {
+ HostDefinition host = new HostDefinition(hostString);
+ InetAddress addr;
+ try {
+ addr = InetAddress.getByName(host.getHostname());
+ } catch (UnknownHostException e) {
+ throw new InvalidHostStringException("Unable to resolve host "
+ + host.getRawHostname(), e);
+ }
+ String dnsName;
+ if (useHostnames) {
+ dnsName = addr.getHostName();
+ } else {
+ dnsName = addr.getHostAddress();
+ }
+ List<DatanodeDetails> found = nodeManager.getNodesByAddress(dnsName);
+ if (found.size() == 0) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " (" + dnsName + ") is not running any datanodes registered"
+ + " with SCM."
+ + " Please check the host name.");
+ } else if (found.size() == 1) {
+ if (host.getPort() != -1 &&
+ !validateDNPortMatch(host.getPort(), found.get(0))) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running a datanode registered with SCM,"
+ + " but the port number doesn't match."
+ + " Please check the port number.");
+ }
+ results.add(found.get(0));
+ } else if (found.size() > 1) {
+ DatanodeDetails match = null;
+ for (DatanodeDetails dn : found) {
+ if (validateDNPortMatch(host.getPort(), dn)) {
+ match = dn;
+ break;
+ }
+ }
+ if (match == null) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running multiple datanodes registered with SCM,"
+ + " but no port numbers match."
+ + " Please check the port number.");
+ }
+ results.add(match);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Check if the passed port is used by the given DatanodeDetails object. If
+ * it is, return true, otherwise return false.
+ * @param port Port number to check if it is used by the datanode
+ * @param dn Datanode to check if it is using the given port
+ * @return True if port is used by the datanode. False otherwise.
+ */
+ private static boolean validateDNPortMatch(int port, DatanodeDetails dn) {
+ for (DatanodeDetails.Port p : dn.getPorts()) {
+ if (p.getValue() == port) {
+ return true;
+ }
+ }
+ return false;
+ }
+ static class HostDefinition {
+ private String rawHostname;
+ private String hostname;
+ private int port;
+
+ HostDefinition(String hostname) throws InvalidHostStringException {
+ this.rawHostname = hostname;
+ parseHostname();
+ }
+
+ public String getRawHostname() {
+ return rawHostname;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ private void parseHostname() throws InvalidHostStringException {
+ try {
+ // A URI *must* have a scheme, so just create a fake one
+ URI uri = new URI("empty://" + rawHostname.trim());
+ this.hostname = uri.getHost();
+ this.port = uri.getPort();
+
+ if (this.hostname == null) {
+ throw new InvalidHostStringException("The string " + rawHostname +
+ " does not contain a value hostname or hostname:port definition");
+ }
+ } catch (URISyntaxException e) {
+ throw new InvalidHostStringException(
+ "Unable to parse the hoststring " + rawHostname, e);
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index f402b9309f..aa0a197dc1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -714,6 +715,14 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
.setDecommissionScmResponse(decommissionScm(
request.getDecommissionScmRequest()))
.build();
+ case DatanodeDiskBalancerInfo:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerInfoResponse(getDatanodeDiskBalancerInfo(
+ request.getDatanodeDiskBalancerInfoRequest(),
+ request.getVersion()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1285,6 +1294,28 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
public DecommissionScmResponseProto decommissionScm(
DecommissionScmRequestProto request) throws IOException {
return impl.decommissionScm(
- request.getScmId());
+ request.getScmId());
+ }
+
+ public DatanodeDiskBalancerInfoResponseProto getDatanodeDiskBalancerInfo(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerInfoRequestProto request, int clientVersion)
+ throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> infoProtoList;
+ switch (request.getInfoType()) {
+ case report:
+ infoProtoList = impl.getDiskBalancerReport(request.getCount(),
+ clientVersion);
+ break;
+ case status:
+ infoProtoList = impl.getDiskBalancerStatus(
+ Optional.of(request.getHostsList()), clientVersion);
+ break;
+ default:
+ infoProtoList = null;
+ }
+ return DatanodeDiskBalancerInfoResponseProto.newBuilder()
+ .addAllInfo(infoProtoList)
+ .build();
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 13bef8590b..dacdbe0e3f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1273,6 +1273,56 @@ public class SCMClientProtocolServer implements
ContainerID.valueOf(startContainerID), count, state);
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser(), true);
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerReport(count,
+ clientVersion);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser(), true);
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ clientVersion);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
/**
* Queries a list of Node that match a set of statuses.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 046be68760..3e55d7ded0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.security.SecretKeyManagerService;
import org.apache.hadoop.hdds.scm.security.RootCARotationManager;
@@ -259,6 +260,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private FinalizationManager finalizationManager;
private HDDSLayoutVersionManager scmLayoutVersionManager;
private LeaseManager<Object> leaseManager;
+ private DiskBalancerManager diskBalancerManager;
private SCMMetadataStore scmMetadataStore;
private CertificateStore certificateStore;
@@ -855,6 +857,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
.setSCMDBTransactionBuffer(scmHAManager.getDBTransactionBuffer())
.setRatisServer(scmHAManager.getRatisServer())
.build();
+ diskBalancerManager = new DiskBalancerManager(conf, eventQueue, scmContext,
+ scmNodeManager);
}
/**
@@ -2008,6 +2012,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return serviceManager;
}
+ public DiskBalancerManager getDiskBalancerManager() {
+ return diskBalancerManager;
+ }
+
/**
* Force SCM out of safe mode.
*/
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
new file mode 100644
index 0000000000..541c9764b7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Unit tests for the DiskBalancer manager.
+ */
+
+public class TestDiskBalancerManager {
+
+ private DiskBalancerManager diskBalancerManager;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private String storageDir;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ storageDir = GenericTestUtils.getTempPath(
+ TestDiskBalancerManager.class.getSimpleName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ nodeManager = new MockNodeManager(true, 3);
+ diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
+ SCMContext.emptyContext(), nodeManager);
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ diskBalancerManager.getDiskBalancerReport(2,
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(2, reportProtoList.size());
+ Assertions.assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+
+ // Simulate users asking all status of 3 datanodes
+ List<String> dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).collect(
+ Collectors.toList());
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(3, statusProtoList.size());
+
+ // Simulate users asking status of 1 datanodes
+ dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).limit(1).collect(
+ Collectors.toList());
+
+ statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(1, statusProtoList.size());
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index d07e696e7e..b16b4dc18a 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -563,4 +563,39 @@ public class ContainerOperationClient implements ScmClient {
return storageContainerLocationClient.decommissionScm(scmId);
}
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerReport(count,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
+ hosts);
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.stopDiskBalancer(hosts);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerStatus(hosts,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
+ bandwidth, hosts);
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
new file mode 100644
index 0000000000..c047cd3ee0
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * This class tests disk balancer operations.
+ */
+@Timeout(300)
+public class TestDiskBalancer {
+
+ private static ScmClient storageClient;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration ozoneConf;
+
+ @BeforeAll
+ public static void setup() throws Exception {
+ ozoneConf = new OzoneConfiguration();
+ ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
+ cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
+ storageClient = new ContainerOperationClient(ozoneConf);
+ cluster.waitForClusterToBeReady();
+
+ for (DatanodeDetails dn: cluster.getStorageContainerManager()
+ .getScmNodeManager().getAllNodes()) {
+ ((DatanodeInfo) dn).updateStorageReports(
+ HddsTestUtils.getRandomNodeReport(3, 1).getStorageReportList());
+ }
+ }
+
+ @AfterAll
+ public static void cleanup() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ storageClient.getDiskBalancerReport(2);
+
+ assertEquals(2, reportProtoList.size());
+ assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ // TODO: Test status command with datanodes in balancing
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 04/07: HDDS-7234. Add a common option for DiskBalancer commands (#3762)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 80ded7c4196633e0dc7af0fd5150af9609719339
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 22 10:00:39 2022 +0800
HDDS-7234. Add a common option for DiskBalancer commands (#3762)
---
.../scm/cli/datanode/DiskBalancerCommands.java | 18 ++---
.../cli/datanode/DiskBalancerCommonOptions.java | 84 ++++++++++++++++++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 27 +++----
.../cli/datanode/DiskBalancerStatusSubcommand.java | 7 +-
.../cli/datanode/DiskBalancerStopSubcommand.java | 29 +++-----
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 27 +++----
6 files changed, 123 insertions(+), 69 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
index 649ee59eaa..dd70bbb491 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -38,36 +38,38 @@ import java.util.concurrent.Callable;
* [ -t/--threshold {@literal <threshold>}]
* [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
* [ -p/--parallelThread {@literal <parallelThread>}]
- * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ -a/--all {@literal <alldatanodes>}]
+ * [ -d/--datanodes {@literal <datanodes>}]
* [ {@literal <hosts>}]
* Examples:
- * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -d {@literal <hosts>}
* start balancer with default values in the configuration on specified
* datanodes
* ozone admin datanode diskbalancer start -a
* start balancer with default values in the configuration on all
* datanodes in the cluster
- * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -t 5 -d {@literal <hosts>}
* start balancer with a threshold of 5%
- * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -b 20 -d {@literal <hosts>}
* start balancer with maximum 20MB/s diskbandwidth
- * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -p 5 -d {@literal <hosts>}
* start balancer with 5 parallel thread on each datanode
* To stop:
* ozone admin datanode diskbalancer stop -a
* stop diskblancer on all datanodes
- * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * ozone admin datanode diskbalancer stop -d {@literal <hosts>};
* stop diskblancer on all datanodes
* To update:
* ozone admin datanode diskbalancer update -a
* update diskblancer configuration on all datanodes
- * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * ozone admin datanode diskbalancer update -d {@literal <hosts>};
* update diskblancer configuration on all datanodes
* To get report:
* ozone admin datanode diskbalancer report -c 10
* retrieve at most 10 datanodes that needs diskbalance most
* To get status:
- * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * ozone admin datanode diskbalancer status -s RUNNING -d
+ * {@literal <hosts>}
* return the diskbalancer status on datanodes where diskbalancer are in
* Running state
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
new file mode 100644
index 0000000000..744de5169f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import picocli.CommandLine;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Common options for DiskBalancer commands.
+ */
+public class DiskBalancerCommonOptions {
+
+ @CommandLine.Option(names = {"-a", "--all"},
+ description = "Run commands on all datanodes.")
+ private boolean allHosts;
+
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Run commands on specific datanodes, the content can be " +
+ "a list of hostnames or IPs. " +
+ "Examples: hostname1,hostname2,hostname3 or ip1,ip2,ip3")
+ private List<String> datanodes = new ArrayList<>();
+
+ /**
+ * Check the common options of DiskBalancerCommand.
+ * @return if the check passed
+ */
+ public boolean check() {
+ if (datanodes.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify at least " +
+ "one datanode or use \"-a(--all)\" to start diskBalancer " +
+ "on all datanodes");
+ return false;
+ }
+ if (datanodes.size() != 0 && allHosts) {
+ System.out.println("Invalid option selection. " +
+ "Use either \"-a(--all)\" or \"-d(--datanodes)\".");
+ return false;
+ }
+ return true;
+ }
+
+ public String getHostString() {
+ return isAllHosts() ? "All datanodes" : String.join("\n", getDatanodes());
+ }
+
+ public Optional<List<String>> getSpecifiedDatanodes() {
+ return getDatanodes().size() == 0 ?
+ Optional.empty() : Optional.of(getDatanodes());
+ }
+
+ public boolean isAllHosts() {
+ return allHosts;
+ }
+
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+
+ public List<String> getDatanodes() {
+ return datanodes;
+ }
+
+ public void setDatanodes(List<String> datanodes) {
+ this.datanodes = datanodes;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
index e3862ba23f..f58c7a1f84 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified. Please specify " +
- "\"--allDatanodes\" to start diskBalancer on all datanodes");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Start DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -95,6 +84,6 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
index 6a61876479..84dd3abe95 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -21,9 +21,9 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
import java.util.ArrayList;
@@ -48,10 +48,11 @@ public class DiskBalancerStatusSubcommand extends ScmSubcommand {
new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
@Option(names = {"-s", "--state"},
- description = "RUNNING, STOPPED, UNKNOWN")
+ description = "RUNNING, STOPPED, UNKNOWN. Default state is RUNNING.")
private String state = "RUNNING";
- @Parameters(description = "List of fully qualified host names")
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Get diskBalancer status on specific datanodes.")
private List<String> hosts = new ArrayList<>();
@Override
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
index 9479c030a6..f63f880a44 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -22,14 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
/**
* Handler to stop disk balancer.
@@ -40,29 +37,21 @@ import java.util.Optional;
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class DiskBalancerStopSubcommand extends ScmSubcommand {
- @Option(names = {"-a", "--allDatanodes"},
- description = "Stop diskBalancer on all datanodes.")
- private boolean allHosts;
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
+ if (!commonOptions.check()) {
return;
}
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
- return;
- }
- List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
- Optional.empty() : Optional.of(hosts));
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Stopping DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
System.err.println("Error: " + error.getHostname() + ": "
@@ -77,6 +66,6 @@ public class DiskBalancerStopSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
index b19fb6a53a..28ed4fd325 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
- parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ parallelThread, commonOptions.getSpecifiedDatanodes());
System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -93,6 +82,6 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 05/07: HDDS-7383. Basic framework of DiskBalancerService (#3874)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a1b69e1da9d2cc5de00f3e5538c738e68b60bb29
Author: Symious <14...@users.noreply.github.com>
AuthorDate: Thu Feb 23 17:10:58 2023 +0800
HDDS-7383. Basic framework of DiskBalancerService (#3874)
---
.../scm/storage/DiskBalancerConfiguration.java | 85 ++++-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../hadoop/hdds/fs/MockSpaceUsageCheckFactory.java | 16 +
.../commandhandler/DiskBalancerCommandHandler.java | 39 +-
.../container/diskbalancer/DiskBalancerInfo.java | 130 +++++++
.../diskbalancer/DiskBalancerService.java | 397 +++++++++++++++++++++
.../diskbalancer/DiskBalancerServiceMetrics.java | 138 +++++++
.../diskbalancer/DiskBalancerVersion.java | 79 ++++
.../container/diskbalancer/DiskBalancerYaml.java | 165 +++++++++
.../ozone/container/diskbalancer/package-info.java | 22 ++
.../ozone/container/ozoneimpl/OzoneContainer.java | 26 +-
.../protocol/commands/DiskBalancerCommand.java | 25 +-
.../diskbalancer/DiskBalancerServiceTestImpl.java | 104 ++++++
.../diskbalancer/TestDiskBalancerService.java | 173 +++++++++
.../diskbalancer/TestDiskBalancerYaml.java | 53 +++
...inerLocationProtocolClientSideTranslatorPB.java | 7 +-
.../src/main/proto/ScmAdminProtocol.proto | 8 +-
.../interface-client/src/main/proto/hdds.proto | 6 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +-
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 141 ++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 32 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 2 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 2 +-
24 files changed, 1624 insertions(+), 37 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index e460e0b6eb..ed40fa30f6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -28,6 +28,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Duration;
+import java.util.Optional;
+
+import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE;
+
/**
* This class contains configuration values for the DiskBalancer.
*/
@@ -36,6 +41,17 @@ public final class DiskBalancerConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerConfiguration.class);
+ @Config(key = "info.dir", type = ConfigType.STRING,
+ defaultValue = "", tags = {ConfigTag.DISKBALANCER},
+ description = "The path where datanode diskBalancer's conf is to be " +
+ "written to. if this property is not defined, ozone will fall " +
+ "back to use metadata directory instead.")
+ private String infoDir;
+
+ public String getDiskBalancerInfoDir() {
+ return infoDir;
+ }
+
@Config(key = "volume.density.threshold", type = ConfigType.DOUBLE,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "Threshold is a percentage in the range of 0 to 100. A " +
@@ -55,6 +71,72 @@ public final class DiskBalancerConfiguration {
description = "The max parallel balance thread count.")
private int parallelThread = 5;
+ @Config(key = "should.run.default",
+ defaultValue = "false",
+ type = ConfigType.BOOLEAN,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description =
+ "If DiskBalancer fails to get information from diskbalancer.info, " +
+ "it will choose this value to decide if this service should be " +
+ "running."
+ )
+ private boolean diskBalancerShouldRun = false;
+
+ public boolean getDiskBalancerShouldRun() {
+ return diskBalancerShouldRun;
+ }
+
+ public void setDiskBalancerShouldRun(boolean shouldRun) {
+ this.diskBalancerShouldRun = shouldRun;
+ }
+
+ @Config(key = "service.interval",
+ defaultValue = "60s",
+ type = ConfigType.TIME,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description = "Time interval of the Datanode DiskBalancer service. " +
+ "The Datanode will check the service periodically and update " +
+ "the config and running status for DiskBalancer service. " +
+ "Unit could be defined with postfix (ns,ms,s,m,h,d). "
+ )
+ private long diskBalancerInterval = Duration.ofSeconds(60).toMillis();
+
+ public Duration getDiskBalancerInterval() {
+ return Duration.ofMillis(diskBalancerInterval);
+ }
+
+ public void setDiskBalancerInterval(Duration duration) {
+ this.diskBalancerInterval = duration.toMillis();
+ }
+
+ @Config(key = "service.timeout",
+ defaultValue = "300s",
+ type = ConfigType.TIME,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description = "Timeout for the Datanode DiskBalancer service. "
+ + "Unit could be defined with postfix (ns,ms,s,m,h,d). "
+ )
+ private long diskBalancerTimeout = Duration.ofSeconds(300).toMillis();
+
+ public Duration getDiskBalancerTimeout() {
+ return Duration.ofMillis(diskBalancerTimeout);
+ }
+
+ public void setDiskBalancerTimeout(Duration duration) {
+ this.diskBalancerTimeout = duration.toMillis();
+ }
+
+ public DiskBalancerConfiguration() {
+ }
+
+ public DiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread) {
+ threshold.ifPresent(aDouble -> this.threshold = aDouble);
+ bandwidthInMB.ifPresent(aLong -> this.diskBandwidthInMB = aLong);
+ parallelThread.ifPresent(integer -> this.parallelThread = integer);
+ }
+
/**
* Gets the threshold value for DiskBalancer.
*
@@ -86,7 +168,8 @@ public final class DiskBalancerConfiguration {
*
* @return max disk bandwidth per second
*/
- public double getDiskBandwidthInMB() {
+
+ public long getDiskBandwidthInMB() {
return diskBandwidthInMB;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 653af11ce8..e6ba9ef90c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -241,6 +241,9 @@ public final class OzoneConsts {
*/
public static final String OZONE_SCM_DATANODE_ID_FILE_DEFAULT = "datanode.id";
+ public static final String
+ OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT = "diskBalancer.info";
+
// The ServiceListJSONServlet context attribute where OzoneManager
// instance gets stored.
public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
index 3b2e6415c2..eaeb3c1b4d 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
@@ -50,6 +50,22 @@ public final class MockSpaceUsageCheckFactory {
}
}
+ /**
+ * An implementation that never checks space usage but reports basically
+ * 512G free space. Neither does it persist space usage info.
+ */
+ public static class HalfTera implements SpaceUsageCheckFactory {
+ @Override
+ public SpaceUsageCheckParams paramsFor(File dir) {
+ return new SpaceUsageCheckParams(dir,
+ MockSpaceUsageSource.fixed(512L * 1024 * 1024 * 1024,
+ 512L * 1024 * 1024 * 1024),
+ Duration.ZERO,
+ SpaceUsagePersistence.None.INSTANCE
+ );
+ }
+ }
+
private MockSpaceUsageCheckFactory() {
throw new UnsupportedOperationException("no instances");
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
index 37f275dbb4..6ff745cafe 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
@@ -16,15 +16,21 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerInfo;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -56,7 +62,38 @@ public class DiskBalancerCommandHandler implements CommandHandler {
public void handle(SCMCommand command, OzoneContainer ozoneContainer,
StateContext context, SCMConnectionManager connectionManager) {
invocationCount.incrementAndGet();
- // TODO: Do start/stop/update operation
+ final long startTime = Time.monotonicNow();
+ DiskBalancerCommand diskBalancerCommand = (DiskBalancerCommand) command;
+
+ final HddsProtos.DiskBalancerOpType opType =
+ diskBalancerCommand.getOpType();
+ final DiskBalancerConfiguration diskBalancerConf =
+ diskBalancerCommand.getDiskBalancerConfiguration();
+
+ DiskBalancerInfo diskBalancerInfo = ozoneContainer.getDiskBalancerInfo();
+
+ try {
+ switch (opType) {
+ case START:
+ diskBalancerInfo.setShouldRun(true);
+ diskBalancerInfo.updateFromConf(diskBalancerConf);
+ break;
+ case STOP:
+ diskBalancerInfo.setShouldRun(false);
+ break;
+ case UPDATE:
+ diskBalancerInfo.updateFromConf(diskBalancerConf);
+ break;
+ default:
+ throw new IOException("Unexpected type " + opType);
+ }
+ ozoneContainer.getDiskBalancerService().refresh(diskBalancerInfo);
+ } catch (IOException e) {
+ LOG.error("Can't handle command type: {}", opType, e);
+ } finally {
+ long endTime = Time.monotonicNow();
+ totalTime += endTime - startTime;
+ }
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
new file mode 100644
index 0000000000..873c89aad1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+
+import java.util.Objects;
+
+/**
+ * DiskBalancer's information to persist.
+ */
+public class DiskBalancerInfo {
+ private boolean shouldRun;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+ private DiskBalancerVersion version;
+
+ public DiskBalancerInfo(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread) {
+ this(shouldRun, threshold, bandwidthInMB, parallelThread,
+ DiskBalancerVersion.DEFAULT_VERSION);
+ }
+
+ public DiskBalancerInfo(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread, DiskBalancerVersion version) {
+ this.shouldRun = shouldRun;
+ this.threshold = threshold;
+ this.bandwidthInMB = bandwidthInMB;
+ this.parallelThread = parallelThread;
+ this.version = version;
+ }
+
+ public DiskBalancerInfo(boolean shouldRun,
+ DiskBalancerConfiguration diskBalancerConf) {
+ this.shouldRun = shouldRun;
+ this.threshold = diskBalancerConf.getThreshold();
+ this.bandwidthInMB = diskBalancerConf.getDiskBandwidthInMB();
+ this.parallelThread = diskBalancerConf.getParallelThread();
+ this.version = DiskBalancerVersion.DEFAULT_VERSION;
+ }
+
+ public void updateFromConf(DiskBalancerConfiguration diskBalancerConf) {
+ if (threshold != diskBalancerConf.getThreshold()) {
+ setThreshold(diskBalancerConf.getThreshold());
+ }
+ if (bandwidthInMB != diskBalancerConf.getDiskBandwidthInMB()) {
+ setBandwidthInMB(diskBalancerConf.getDiskBandwidthInMB());
+ }
+ if (parallelThread != diskBalancerConf.getParallelThread()) {
+ setParallelThread(diskBalancerConf.getParallelThread());
+ }
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public long getBandwidthInMB() {
+ return bandwidthInMB;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public int getParallelThread() {
+ return parallelThread;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public DiskBalancerVersion getVersion() {
+ return version;
+ };
+
+ public void setVersion(DiskBalancerVersion version) {
+ this.version = version;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DiskBalancerInfo that = (DiskBalancerInfo) o;
+ return shouldRun == that.shouldRun &&
+ Double.compare(that.threshold, threshold) == 0 &&
+ bandwidthInMB == that.bandwidthInMB &&
+ parallelThread == that.parallelThread &&
+ version == that.version;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(shouldRun, threshold, bandwidthInMB, parallelThread,
+ version);
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
new file mode 100644
index 0000000000..75e1660029
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.util.Time;
+import org.apache.ratis.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A per-datanode disk balancing service takes in charge
+ * of moving contains among disks.
+ */
+public class DiskBalancerService extends BackgroundService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerService.class);
+
+ public static final String DISK_BALANCER_DIR = "diskBalancer";
+
+ private static final String DISK_BALANCER_TMP_DIR = "tmp";
+
+ private OzoneContainer ozoneContainer;
+ private final ConfigurationSource conf;
+
+ private boolean shouldRun = false;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+
+ private DiskBalancerVersion version;
+
+ private AtomicLong totalBalancedBytes = new AtomicLong(0L);
+ private AtomicLong balancedBytesInLastWindow = new AtomicLong(0L);
+ private AtomicLong nextAvailableTime = new AtomicLong(Time.monotonicNow());
+
+ private Map<DiskBalancerTask, Integer> inProgressTasks;
+ private Set<Long> inProgressContainers;
+
+ // Every time a container is decided to be moved from Vol A to Vol B,
+ // the size will be deducted from Vol A and added to Vol B.
+ // This map is used to help calculate the expected storage size after
+ // the container balancing finished.
+ private Map<HddsVolume, Long> deltaSizes;
+ private MutableVolumeSet volumeSet;
+
+ private final File diskBalancerInfoFile;
+
+ private DiskBalancerServiceMetrics metrics;
+
+ public DiskBalancerService(OzoneContainer ozoneContainer,
+ long serviceCheckInterval, long serviceCheckTimeout, TimeUnit timeUnit,
+ int workerSize, ConfigurationSource conf) throws IOException {
+ super("DiskBalancerService", serviceCheckInterval, timeUnit, workerSize,
+ serviceCheckTimeout);
+ this.ozoneContainer = ozoneContainer;
+ this.conf = conf;
+
+ String diskBalancerInfoPath = getDiskBalancerInfoPath();
+ Preconditions.checkNotNull(diskBalancerInfoPath);
+ diskBalancerInfoFile = new File(diskBalancerInfoPath);
+
+ inProgressTasks = new ConcurrentHashMap<>();
+ inProgressContainers = ConcurrentHashMap.newKeySet();
+ deltaSizes = new ConcurrentHashMap<>();
+ volumeSet = ozoneContainer.getVolumeSet();
+
+ metrics = DiskBalancerServiceMetrics.create();
+
+ loadDiskBalancerInfo();
+
+ constructTmpDir();
+ }
+
+ /**
+ * Update DiskBalancerService based on new DiskBalancerInfo.
+ * @param diskBalancerInfo
+ * @throws IOException
+ */
+ public void refresh(DiskBalancerInfo diskBalancerInfo) throws IOException {
+ applyDiskBalancerInfo(diskBalancerInfo);
+ }
+
+ private void constructTmpDir() throws IOException {
+ for (HddsVolume volume:
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())) {
+ Path tmpDir = getDiskBalancerTmpDir(volume);
+ try {
+ FileUtils.deleteFully(tmpDir);
+ FileUtils.createDirectories(tmpDir);
+ } catch (IOException ex) {
+ LOG.warn("Can not reconstruct tmp directory under volume {}", volume,
+ ex);
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * If the diskBalancer.info file exists, load the file. If not exists,
+ * return the default config.
+ * @throws IOException
+ */
+ private void loadDiskBalancerInfo() throws IOException {
+ DiskBalancerInfo diskBalancerInfo;
+ try {
+ if (diskBalancerInfoFile.exists()) {
+ diskBalancerInfo = readDiskBalancerInfoFile(diskBalancerInfoFile);
+ } else {
+ boolean shouldRunDefault =
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getDiskBalancerShouldRun();
+ diskBalancerInfo = new DiskBalancerInfo(shouldRunDefault,
+ new DiskBalancerConfiguration());
+ }
+ } catch (IOException e) {
+ LOG.warn("Can not load diskBalancerInfo from diskBalancer.info file. " +
+ "Falling back to default configs", e);
+ throw e;
+ }
+
+ applyDiskBalancerInfo(diskBalancerInfo);
+ }
+
+ private void applyDiskBalancerInfo(DiskBalancerInfo diskBalancerInfo)
+ throws IOException {
+ // First store in local file, then update in memory variables
+ writeDiskBalancerInfoTo(diskBalancerInfo, diskBalancerInfoFile);
+
+ setShouldRun(diskBalancerInfo.isShouldRun());
+ setThreshold(diskBalancerInfo.getThreshold());
+ setBandwidthInMB(diskBalancerInfo.getBandwidthInMB());
+ setParallelThread(diskBalancerInfo.getParallelThread());
+ setVersion(diskBalancerInfo.getVersion());
+
+ // Default executorService is ScheduledThreadPoolExecutor, so we can
+ // update the poll size by setting corePoolSize.
+ if ((getExecutorService() instanceof ScheduledThreadPoolExecutor)) {
+ ((ScheduledThreadPoolExecutor) getExecutorService())
+ .setCorePoolSize(parallelThread);
+ }
+ }
+
+ private String getDiskBalancerInfoPath() {
+ String diskBalancerInfoDir =
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getDiskBalancerInfoDir();
+ if (Strings.isNullOrEmpty(diskBalancerInfoDir)) {
+ File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
+ if (metaDirPath == null) {
+ // this means meta data is not found, in theory should not happen at
+ // this point because should've failed earlier.
+ throw new IllegalArgumentException("Unable to locate meta data" +
+ "directory when getting datanode disk balancer file path");
+ }
+ diskBalancerInfoDir = metaDirPath.toString();
+ }
+ // Use default datanode disk balancer file name for file path
+ return new File(diskBalancerInfoDir,
+ OzoneConsts.OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT)
+ .toString();
+ }
+
+ /**
+ * Read {@link DiskBalancerInfo} from a local info file.
+ *
+ * @param path DiskBalancerInfo file local path
+ * @return {@link DatanodeDetails}
+ * @throws IOException If the conf file is malformed or other I/O exceptions
+ */
+ private synchronized DiskBalancerInfo readDiskBalancerInfoFile(
+ File path) throws IOException {
+ if (!path.exists()) {
+ throw new IOException("DiskBalancerConf file not found.");
+ }
+ try {
+ return DiskBalancerYaml.readDiskBalancerInfoFile(path);
+ } catch (IOException e) {
+ LOG.warn("Error loading DiskBalancerInfo yaml from {}",
+ path.getAbsolutePath(), e);
+ throw new IOException("Failed to parse DiskBalancerInfo from "
+ + path.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Persistent a {@link DiskBalancerInfo} to a local file.
+ *
+ * @throws IOException when read/write error occurs
+ */
+ private synchronized void writeDiskBalancerInfoTo(
+ DiskBalancerInfo diskBalancerInfo, File path)
+ throws IOException {
+ if (path.exists()) {
+ if (!path.delete() || !path.createNewFile()) {
+ throw new IOException("Unable to overwrite the DiskBalancerInfo file.");
+ }
+ } else {
+ if (!path.getParentFile().exists() &&
+ !path.getParentFile().mkdirs()) {
+ throw new IOException("Unable to create DiskBalancerInfo directories.");
+ }
+ }
+ DiskBalancerYaml.createDiskBalancerInfoFile(diskBalancerInfo, path);
+ }
+
+
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public void setVersion(DiskBalancerVersion version) {
+ this.version = version;
+ }
+
+ public DiskBalancerInfo getDiskBalancerInfo() {
+ return new DiskBalancerInfo(shouldRun, threshold, bandwidthInMB,
+ parallelThread, version);
+ }
+
+ public DiskBalancerReportProto getDiskBalancerReportProto() {
+ DiskBalancerReportProto.Builder builder =
+ DiskBalancerReportProto.newBuilder();
+ return builder.setIsRunning(shouldRun)
+ .setBalancedBytes(totalBalancedBytes.get())
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(threshold)
+ .setDiskBandwidthInMB(bandwidthInMB)
+ .setParallelThread(parallelThread)
+ .build())
+ .build();
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue queue = new BackgroundTaskQueue();
+
+ if (!shouldRun) {
+ return queue;
+ }
+ metrics.incrRunningLoopCount();
+
+ if (shouldDelay()) {
+ metrics.incrIdleLoopExceedsBandwidthCount();
+ return queue;
+ }
+
+ int availableTaskCount = parallelThread - inProgressTasks.size();
+ if (availableTaskCount <= 0) {
+ LOG.info("No available thread for disk balancer service. " +
+ "Current thread count is {}.", parallelThread);
+ return queue;
+ }
+
+ // TODO: Implementation for choose tasks
+
+ if (queue.isEmpty()) {
+ metrics.incrIdleLoopNoAvailableVolumePairCount();
+ }
+ return queue;
+ }
+
+ private boolean shouldDelay() {
+ // We should wait for next AvailableTime.
+ if (Time.monotonicNow() <= nextAvailableTime.get()) {
+ return true;
+ }
+ // Calculate the next AvailableTime based on bandwidth
+ long bytesBalanced = balancedBytesInLastWindow.getAndSet(0L);
+
+ final int megaByte = 1024 * 1024;
+
+ // converting disk bandwidth in byte/millisec
+ float bytesPerMillisec = bandwidthInMB * megaByte / 1000f;
+ nextAvailableTime.set(Time.monotonicNow() +
+ ((long) (bytesBalanced / bytesPerMillisec)));
+ return false;
+ }
+
+ private class DiskBalancerTask implements BackgroundTask {
+
+ private HddsVolume sourceVolume;
+ private HddsVolume destVolume;
+ private ContainerData containerData;
+
+ DiskBalancerTask(ContainerData containerData,
+ HddsVolume sourceVolume, HddsVolume destVolume) {
+ this.containerData = containerData;
+ this.sourceVolume = sourceVolume;
+ this.destVolume = destVolume;
+ }
+
+ @Override
+ public BackgroundTaskResult call() {
+ // TODO: Details of handling tasks
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+
+ @Override
+ public int getPriority() {
+ return BackgroundTask.super.getPriority();
+ }
+
+ private void postCall() {
+ inProgressContainers.remove(containerData.getContainerID());
+ deltaSizes.put(sourceVolume, deltaSizes.get(sourceVolume) +
+ containerData.getBytesUsed());
+ deltaSizes.put(destVolume, deltaSizes.get(destVolume)
+ - containerData.getBytesUsed());
+ }
+ }
+
+ private Path getDiskBalancerTmpDir(HddsVolume hddsVolume) {
+ return Paths.get(hddsVolume.getVolumeRootDir())
+ .resolve(DISK_BALANCER_TMP_DIR).resolve(DISK_BALANCER_DIR);
+ }
+
+ public boolean isBalancingContainer(long containerId) {
+ return inProgressContainers.contains(containerId);
+ }
+
+ public DiskBalancerServiceMetrics getMetrics() {
+ return metrics;
+ }
+
+ @VisibleForTesting
+ public void setBalancedBytesInLastWindow(long bytes) {
+ this.balancedBytesInLastWindow.set(bytes);
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ if (metrics != null) {
+ DiskBalancerServiceMetrics.unRegister();
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java
new file mode 100644
index 0000000000..3cf087b334
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+
+/**
+ * Metrics related to DiskBalancer Service running on Datanode.
+ */
+@Metrics(name = "DiskBalancerService Metrics", about = "Metrics related to "
+ + "background disk balancer service on Datanode", context = "dfs")
+public final class DiskBalancerServiceMetrics {
+
+ private static DiskBalancerServiceMetrics instance;
+ public static final String SOURCE_NAME =
+ DiskBalancerServiceMetrics.class.getSimpleName();
+
+ @Metric(about = "The number of successful balance job")
+ private MutableCounterLong successCount;
+
+ @Metric(about = "The total bytes for successfully balanced job.")
+ private MutableCounterLong successBytes;
+
+ @Metric(about = "The number of failed balance job.")
+ private MutableCounterLong failureCount;
+
+ @Metric(about = "The number of total running loop")
+ private MutableCounterLong runningLoopCount;
+
+ @Metric(about = "The number of idle loop that can not generate volume pair.")
+ private MutableCounterLong idleLoopNoAvailableVolumePairCount;
+
+ @Metric(about = "The number of idle loop due to bandwidth limits.")
+ private MutableCounterLong idleLoopExceedsBandwidthCount;
+
+ private DiskBalancerServiceMetrics() {
+ }
+
+ public static DiskBalancerServiceMetrics create() {
+ if (instance == null) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ instance = ms.register(SOURCE_NAME, "DiskBalancerService",
+ new DiskBalancerServiceMetrics());
+ }
+
+ return instance;
+ }
+
+ /**
+ * Unregister the metrics instance.
+ */
+ public static void unRegister() {
+ instance = null;
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ ms.unregisterSource(SOURCE_NAME);
+ }
+
+ public void incrSuccessCount(long count) {
+ this.successCount.incr(count);
+ }
+
+ public void incrSuccessBytes(long bytes) {
+ this.successBytes.incr(bytes);
+ }
+
+ public void incrFailureCount() {
+ this.failureCount.incr();
+ }
+
+ public void incrRunningLoopCount() {
+ this.runningLoopCount.incr();
+ }
+
+ public void incrIdleLoopNoAvailableVolumePairCount() {
+ this.idleLoopNoAvailableVolumePairCount.incr();
+ }
+
+ public void incrIdleLoopExceedsBandwidthCount() {
+ this.idleLoopExceedsBandwidthCount.incr();
+ }
+
+ public long getSuccessCount() {
+ return successCount.value();
+ }
+
+ public long getSuccessBytes() {
+ return successBytes.value();
+ }
+
+ public long getFailureCount() {
+ return failureCount.value();
+ }
+
+ public long getRunningLoopCount() {
+ return runningLoopCount.value();
+ }
+
+ public long getIdleLoopNoAvailableVolumePairCount() {
+ return idleLoopNoAvailableVolumePairCount.value();
+ }
+
+ public long getIdleLoopExceedsBandwidthCount() {
+ return idleLoopExceedsBandwidthCount.value();
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("successCount = " + successCount.value()).append("\t")
+ .append("successBytes = " + successBytes.value()).append("\t")
+ .append("failureCount = " + failureCount.value()).append("\t")
+ .append("idleLoopNoAvailableVolumePairCount = " +
+ idleLoopNoAvailableVolumePairCount.value()).append("\t")
+ .append("idleLoopExceedsBandwidthCount = " +
+ idleLoopExceedsBandwidthCount.value());
+ return buffer.toString();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java
new file mode 100644
index 0000000000..28da8993c1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Defines versions for the DiskBalancerService.
+ */
+public enum DiskBalancerVersion {
+ ONE(1, "First Version") {
+ };
+
+ private final int version;
+ private final String description;
+
+ public static final DiskBalancerVersion
+ DEFAULT_VERSION = DiskBalancerVersion.ONE;
+
+ private static final List<DiskBalancerVersion> DISK_BALANCER_VERSIONS =
+ ImmutableList.copyOf(values());
+
+ DiskBalancerVersion(int version, String description) {
+ this.version = version;
+ this.description = description;
+ }
+
+ public static DiskBalancerVersion getDiskBalancerVersion(int version) {
+ for (DiskBalancerVersion diskBalancerVersion :
+ DISK_BALANCER_VERSIONS) {
+ if (diskBalancerVersion.getVersion() == version) {
+ return diskBalancerVersion;
+ }
+ }
+ return null;
+ }
+
+ public static DiskBalancerVersion getDiskBalancerVersion(String versionStr) {
+ for (DiskBalancerVersion diskBalancerVersion :
+ DISK_BALANCER_VERSIONS) {
+ if (diskBalancerVersion.toString().equalsIgnoreCase(versionStr)) {
+ return diskBalancerVersion;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @return version number.
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * @return description.
+ */
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
new file mode 100644
index 0000000000..d16eb65747
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Class for creating diskbalancer.info file in yaml format.
+ */
+
+public final class DiskBalancerYaml {
+
+ private DiskBalancerYaml() {
+ // static helper methods only, no state.
+ }
+
+ /**
+ * Creates a yaml file to store DiskBalancer info.
+ *
+ * @param diskBalancerInfo {@link DiskBalancerInfo}
+ * @param path Path to diskBalancer.info file
+ */
+ public static void createDiskBalancerInfoFile(
+ DiskBalancerInfo diskBalancerInfo, File path)
+ throws IOException {
+ DumperOptions options = new DumperOptions();
+ options.setPrettyFlow(true);
+ options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW);
+ Yaml yaml = new Yaml(options);
+
+ try (Writer writer = new OutputStreamWriter(
+ new FileOutputStream(path), StandardCharsets.UTF_8)) {
+ yaml.dump(getDiskBalancerInfoYaml(diskBalancerInfo), writer);
+ }
+ }
+
+ /**
+ * Read DiskBalancerConfiguration from file.
+ */
+ public static DiskBalancerInfo readDiskBalancerInfoFile(File path)
+ throws IOException {
+ DiskBalancerInfo diskBalancerInfo;
+
+ try (FileInputStream inputFileStream = new FileInputStream(path)) {
+ Yaml yaml = new Yaml();
+ DiskBalancerInfoYaml diskBalancerInfoYaml;
+ try {
+ diskBalancerInfoYaml =
+ yaml.loadAs(inputFileStream, DiskBalancerInfoYaml.class);
+ } catch (Exception e) {
+ throw new IOException("Unable to parse yaml file.", e);
+ }
+
+ diskBalancerInfo = new DiskBalancerInfo(
+ diskBalancerInfoYaml.isShouldRun(),
+ diskBalancerInfoYaml.getThreshold(),
+ diskBalancerInfoYaml.getBandwidthInMB(),
+ diskBalancerInfoYaml.getParallelThread(),
+ DiskBalancerVersion.getDiskBalancerVersion(
+ diskBalancerInfoYaml.version));
+ }
+
+ return diskBalancerInfo;
+ }
+
+ /**
+ * Datanode DiskBalancer Info to be written to the yaml file.
+ */
+ public static class DiskBalancerInfoYaml {
+ private boolean shouldRun;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+
+ private int version;
+
+ public DiskBalancerInfoYaml() {
+ // Needed for snake-yaml introspection.
+ }
+
+ private DiskBalancerInfoYaml(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread, int version) {
+ this.shouldRun = shouldRun;
+ this.threshold = threshold;
+ this.bandwidthInMB = bandwidthInMB;
+ this.parallelThread = parallelThread;
+ this.version = version;
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public long getBandwidthInMB() {
+ return this.bandwidthInMB;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public int getParallelThread() {
+ return this.parallelThread;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ public int getVersion() {
+ return this.version;
+ }
+ }
+
+ private static DiskBalancerInfoYaml getDiskBalancerInfoYaml(
+ DiskBalancerInfo diskBalancerInfo) {
+
+ return new DiskBalancerInfoYaml(
+ diskBalancerInfo.isShouldRun(),
+ diskBalancerInfo.getThreshold(),
+ diskBalancerInfo.getBandwidthInMB(),
+ diskBalancerInfo.getParallelThread(),
+ diskBalancerInfo.getVersion().getVersion());
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java
new file mode 100644
index 0000000000..d8ba71eb9d
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ This package contains classes related to the DiskBalancer service.
+ */
+package org.apache.hadoop.ozone.container.diskbalancer;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index f81b58e317..a79a160e81 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
@@ -56,6 +57,8 @@ import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume.VolumeType;
import org.apache.hadoop.ozone.container.common.volume.StorageVolumeChecker;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerInfo;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerService;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
import org.apache.hadoop.ozone.container.replication.ContainerImporter;
import org.apache.hadoop.ozone.container.replication.ReplicationServer;
@@ -118,6 +121,7 @@ public class OzoneContainer {
private final StaleRecoveringContainerScrubbingService
recoveringContainerScrubbingService;
private final GrpcTlsConfig tlsClientConfig;
+ private final DiskBalancerService diskBalancerService;
private final AtomicReference<InitializingStatus> initializingStatus;
private final ReplicationServer replicationServer;
private DatanodeDetails datanodeDetails;
@@ -239,6 +243,15 @@ public class OzoneContainer {
datanodeDetails.threadNamePrefix(),
context.getParent().getReconfigurationHandler());
+ Duration diskBalancerSvcInterval = conf.getObject(
+ DiskBalancerConfiguration.class).getDiskBalancerInterval();
+ Duration diskBalancerSvcTimeout = conf.getObject(
+ DiskBalancerConfiguration.class).getDiskBalancerTimeout();
+ diskBalancerService =
+ new DiskBalancerService(this, diskBalancerSvcInterval.toMillis(),
+ diskBalancerSvcTimeout.toMillis(), TimeUnit.MILLISECONDS, 1,
+ config);
+
Duration recoveringContainerScrubbingSvcInterval = conf.getObject(
DatanodeConfiguration.class).getRecoveringContainerScrubInterval();
@@ -468,6 +481,7 @@ public class OzoneContainer {
hddsDispatcher.init();
hddsDispatcher.setClusterId(clusterId);
blockDeletingService.start();
+ diskBalancerService.start();
recoveringContainerScrubbingService.start();
// mark OzoneContainer as INITIALIZED.
@@ -493,6 +507,7 @@ public class OzoneContainer {
dbVolumeSet.shutdown();
}
blockDeletingService.shutdown();
+ diskBalancerService.shutdown();
recoveringContainerScrubbingService.shutdown();
ContainerMetrics.remove();
}
@@ -592,7 +607,14 @@ public class OzoneContainer {
}
public DiskBalancerReportProto getDiskBalancerReport() {
- // TODO: Return real disk balancer report
- return null;
+ return diskBalancerService.getDiskBalancerReportProto();
+ }
+
+ public DiskBalancerInfo getDiskBalancerInfo() {
+ return diskBalancerService.getDiskBalancerInfo();
+ }
+
+ public DiskBalancerService getDiskBalancerService() {
+ return diskBalancerService;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
index ea97f34ca0..1780877787 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.protocol.commands;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
@@ -29,12 +30,12 @@ import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProt
*/
public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
- private final boolean shouldRun;
+ private final HddsProtos.DiskBalancerOpType opType;
private final DiskBalancerConfiguration diskBalancerConfiguration;
- public DiskBalancerCommand(final boolean shouldRun,
+ public DiskBalancerCommand(final HddsProtos.DiskBalancerOpType opType,
final DiskBalancerConfiguration diskBalancerConfiguration) {
- this.shouldRun = shouldRun;
+ this.opType = opType;
this.diskBalancerConfiguration = diskBalancerConfiguration;
}
@@ -50,22 +51,26 @@ public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
@Override
public DiskBalancerCommandProto getProto() {
- return DiskBalancerCommandProto.newBuilder()
- .setShouldRun(shouldRun)
- .setDiskBalancerConf(diskBalancerConfiguration.toProtobufBuilder())
- .build();
+ DiskBalancerCommandProto.Builder builder = DiskBalancerCommandProto
+ .newBuilder().setOpType(opType);
+ // Stop command don't have diskBalancerConf
+ if (diskBalancerConfiguration != null) {
+ builder.setDiskBalancerConf(
+ diskBalancerConfiguration.toProtobufBuilder());
+ }
+ return builder.build();
}
public static DiskBalancerCommand getFromProtobuf(DiskBalancerCommandProto
diskbalancerCommandProto, ConfigurationSource configuration) {
Preconditions.checkNotNull(diskbalancerCommandProto);
- return new DiskBalancerCommand(diskbalancerCommandProto.getShouldRun(),
+ return new DiskBalancerCommand(diskbalancerCommandProto.getOpType(),
DiskBalancerConfiguration.fromProtobuf(
diskbalancerCommandProto.getDiskBalancerConf(), configuration));
}
- public boolean isShouldRun() {
- return shouldRun;
+ public HddsProtos.DiskBalancerOpType getOpType() {
+ return opType;
}
public DiskBalancerConfiguration getDiskBalancerConfiguration() {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
new file mode 100644
index 0000000000..0d87ac3420
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A test class implementation for {@link DiskBalancerService}.
+ */
+public class DiskBalancerServiceTestImpl extends DiskBalancerService {
+
+ // the service timeout
+ private static final int SERVICE_TIMEOUT_IN_MILLISECONDS = 0;
+
+ // tests only
+ private CountDownLatch latch;
+ private Thread testingThread;
+ private AtomicInteger numOfProcessed = new AtomicInteger(0);
+
+ public DiskBalancerServiceTestImpl(OzoneContainer container,
+ int serviceInterval, ConfigurationSource conf, int threadCount)
+ throws IOException {
+ super(container, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+ TimeUnit.MILLISECONDS, threadCount, conf);
+ }
+
+ public void runBalanceTasks() {
+ if (latch.getCount() > 0) {
+ this.latch.countDown();
+ } else {
+ throw new IllegalStateException("Count already reaches zero");
+ }
+ }
+
+ public boolean isStarted() {
+ return latch != null && testingThread.isAlive();
+ }
+
+ public int getTimesOfProcessed() {
+ return numOfProcessed.get();
+ }
+
+ // Override the implementation to start a single on-call control thread.
+ @Override
+ public void start() {
+ PeriodicalTask svc = new PeriodicalTask();
+ // In test mode, relies on a latch countdown to runDeletingTasks tasks.
+ Runnable r = () -> {
+ while (true) {
+ latch = new CountDownLatch(1);
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ break;
+ }
+ Future<?> future = this.getExecutorService().submit(svc);
+ try {
+ // for tests, we only wait for 3s for completion
+ future.get(3000, TimeUnit.SECONDS);
+ numOfProcessed.incrementAndGet();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return;
+ }
+ }
+ };
+
+ testingThread = new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .build()
+ .newThread(r);
+ testingThread.start();
+ }
+
+ @Override
+ public void shutdown() {
+ testingThread.interrupt();
+ super.shutdown();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
new file mode 100644
index 0000000000..c03f8cab75
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.container.common.TestBlockDeletingService;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * This is a test class for DiskBalancerService.
+ */
+public class TestDiskBalancerService {
+ private File testRoot;
+ private String scmId;
+ private String datanodeUuid;
+ private OzoneConfiguration conf;
+
+ private final ContainerLayoutVersion layout;
+ private final String schemaVersion;
+ private MutableVolumeSet volumeSet;
+
+ public TestDiskBalancerService(ContainerTestVersionInfo versionInfo) {
+ this.layout = versionInfo.getLayout();
+ this.schemaVersion = versionInfo.getSchemaVersion();
+ conf = new OzoneConfiguration();
+ ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
+ }
+
+
+ @BeforeEach
+ public void init() throws IOException {
+ testRoot = GenericTestUtils
+ .getTestDir(TestBlockDeletingService.class.getSimpleName());
+ if (testRoot.exists()) {
+ FileUtils.cleanDirectory(testRoot);
+ }
+ scmId = UUID.randomUUID().toString();
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY,
+ generateVolumeLocation(testRoot.getAbsolutePath(), 2));
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testRoot.getAbsolutePath());
+ conf.set("hdds.datanode.du.factory.classname",
+ "org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory$HalfTera");
+ datanodeUuid = UUID.randomUUID().toString();
+ volumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ createDbInstancesForTestIfNeeded(volumeSet, scmId, scmId, conf);
+ }
+
+ @AfterEach
+ public void cleanup() throws IOException {
+ BlockUtils.shutdownCache(conf);
+ FileUtils.deleteDirectory(testRoot);
+ }
+
+ @Timeout(30)
+ @ContainerTestVersionInfo.ContainerTest
+ public void testUpdateService() throws Exception {
+ // Increase volume's usedBytes
+ for (StorageVolume volume : volumeSet.getVolumeMap().values()) {
+ volume.incrementUsedSpace(volume.getCapacity() / 2);
+ }
+
+ ContainerSet containerSet = new ContainerSet(1000);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ KeyValueHandler keyValueHandler =
+ new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
+ metrics, c -> {
+ });
+ DiskBalancerServiceTestImpl svc =
+ getDiskBalancerService(containerSet, conf, keyValueHandler, null, 1);
+
+ // Set a low bandwidth to delay job
+ svc.setShouldRun(true);
+ svc.setThreshold(10.0d);
+ svc.setBandwidthInMB(1L);
+ svc.setParallelThread(5);
+ svc.setVersion(DiskBalancerVersion.DEFAULT_VERSION);
+
+ svc.start();
+
+ assertTrue(svc.getDiskBalancerInfo().isShouldRun());
+ assertEquals(10, svc.getDiskBalancerInfo().getThreshold(), 0.0);
+ assertEquals(1, svc.getDiskBalancerInfo().getBandwidthInMB());
+ assertEquals(5, svc.getDiskBalancerInfo().getParallelThread());
+
+ DiskBalancerInfo newInfo = new DiskBalancerInfo(false, 20.0d, 5L, 10);
+ svc.refresh(newInfo);
+
+ assertFalse(svc.getDiskBalancerInfo().isShouldRun());
+ assertEquals(20, svc.getDiskBalancerInfo().getThreshold(), 0.0);
+ assertEquals(5, svc.getDiskBalancerInfo().getBandwidthInMB());
+ assertEquals(10, svc.getDiskBalancerInfo().getParallelThread());
+
+ svc.shutdown();
+ }
+
+ private String generateVolumeLocation(String base, int volumeCount) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < volumeCount; i++) {
+ sb.append(base + "/vol" + i);
+ sb.append(",");
+ }
+ return sb.substring(0, sb.length() - 1);
+ }
+
+ private DiskBalancerServiceTestImpl getDiskBalancerService(
+ ContainerSet containerSet, ConfigurationSource config,
+ KeyValueHandler keyValueHandler, ContainerController controller,
+ int threadCount) throws IOException {
+ OzoneContainer ozoneContainer =
+ mockDependencies(containerSet, keyValueHandler, controller);
+ return new DiskBalancerServiceTestImpl(ozoneContainer, 1000, config,
+ threadCount);
+ }
+
+ private OzoneContainer mockDependencies(ContainerSet containerSet,
+ KeyValueHandler keyValueHandler, ContainerController controller) {
+ OzoneContainer ozoneContainer = mock(OzoneContainer.class);
+ when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
+ when(ozoneContainer.getWriteChannel()).thenReturn(null);
+ ContainerDispatcher dispatcher = mock(ContainerDispatcher.class);
+ when(ozoneContainer.getDispatcher()).thenReturn(dispatcher);
+ when(dispatcher.getHandler(any())).thenReturn(keyValueHandler);
+ when(ozoneContainer.getVolumeSet()).thenReturn(volumeSet);
+ when(ozoneContainer.getController()).thenReturn(controller);
+ return ozoneContainer;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
new file mode 100644
index 0000000000..3b61255b0b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT;
+
+/**
+ * Tests to test DiskBalancer's YAML operation.
+ */
+public class TestDiskBalancerYaml {
+ @Test
+ public void testCreateYaml() throws IOException {
+ boolean shouldRun = true;
+ double threshold = 10;
+ long bandwidthInMB = 100;
+ int parallelThread = 5;
+ DiskBalancerVersion version = DiskBalancerVersion.DEFAULT_VERSION;
+
+ File file = new File(GenericTestUtils.getTestDir(),
+ OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT);
+
+ DiskBalancerInfo info = new DiskBalancerInfo(shouldRun, threshold,
+ bandwidthInMB, parallelThread, version);
+
+ DiskBalancerYaml.createDiskBalancerInfoFile(info, file);
+
+ DiskBalancerInfo newInfo = DiskBalancerYaml.readDiskBalancerInfoFile(file);
+
+ Assertions.assertEquals(info, newInfo);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 7a341820e9..5be9a368f5 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
@@ -1166,7 +1165,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.start)
+ .setOpType(HddsProtos.DiskBalancerOpType.START)
.setConf(confBuilder);
hosts.ifPresent(requestBuilder::addAllHosts);
@@ -1188,7 +1187,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
throws IOException {
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.stop);
+ .setOpType(HddsProtos.DiskBalancerOpType.STOP);
hosts.ifPresent(requestBuilder::addAllHosts);
DatanodeDiskBalancerOpResponseProto response =
@@ -1217,7 +1216,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.update)
+ .setOpType(HddsProtos.DiskBalancerOpType.UPDATE)
.setConf(confBuilder);
hosts.ifPresent(requestBuilder::addAllHosts);
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 807e2634b6..7c77f3d765 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -379,14 +379,8 @@ message DatanodeDiskBalancerInfoResponseProto {
repeated DatanodeDiskBalancerInfoProto info = 1;
}
-enum DatanodeDiskBalancerOpType{
- start = 1;
- stop = 2;
- update = 3;
-}
-
message DatanodeDiskBalancerOpRequestProto {
- required DatanodeDiskBalancerOpType opType = 1;
+ required DiskBalancerOpType opType = 1;
repeated string hosts = 2;
optional DiskBalancerConfigurationProto conf = 3;
}
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 6109ac93fe..78c4e26f9a 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -510,6 +510,12 @@ enum DiskBalancerRunningStatus {
UNKNOWN = 3;
}
+enum DiskBalancerOpType{
+ START = 1;
+ STOP = 2;
+ UPDATE = 3;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 7a265dd0ba..f2f7d537a2 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -452,7 +452,7 @@ message DatanodeDetailsAndReplicaIndexProto {
This command asks the datanode to update diskBalancer status
*/
message DiskBalancerCommandProto {
- required bool shouldRun = 1;
+ required DiskBalancerOpType opType = 1;
optional DiskBalancerConfigurationProto diskBalancerConf = 2;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index c6114c52f5..1af9bfde4c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -24,10 +24,15 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
+import org.apache.ratis.protocol.exceptions.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -126,6 +131,118 @@ public class DiskBalancerManager {
}
}
+ /**
+ * Send startDiskBalancer command to datanodes.
+ * If hosts is not specified, send commands to all healthy datanodes.
+ * @param threshold new configuration of threshold
+ * @param bandwidthInMB new configuration of bandwidthInMB
+ * @param parallelThread new configuration of parallelThread
+ * @param hosts Datanodes that command will apply on
+ * @return Possible errors
+ * @throws IOException
+ */
+ public List<DatanodeAdminError> startDiskBalancer(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ if (nodeManager.getNodeStatus(dn).isHealthy()) {
+ errors.add(new DatanodeAdminError(dn.getHostName(),
+ "Datanode not in healthy state"));
+ continue;
+ }
+ // If command doesn't have configuration change, then we reuse the
+ // latest configuration reported from Datnaodes
+ DiskBalancerConfiguration updateConf = attachDiskBalancerConf(dn,
+ threshold, bandwidthInMB, parallelThread);
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.START, updateConf);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
+ /**
+ * Send stopDiskBalancer command to datanodes
+ * If hosts is not specified, send commands to all datanodes.
+ * @param hosts Datanodes that command will apply on
+ * */
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.STOP, null);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
+
+ /**
+ * Send update DiskBalancerConf command to datanodes.
+ * If hosts is not specified, send commands to all healthy datanodes.
+ * @param threshold new configuration of threshold
+ * @param bandwidthInMB new configuration of bandwidthInMB
+ * @param parallelThread new configuration of parallelThread
+ * @param hosts Datanodes that command will apply on
+ * @return Possible errors
+ * @throws IOException
+ */
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ // If command doesn't have configuration change, then we reuse the
+ // latest configuration reported from Datnaodes
+ DiskBalancerConfiguration updateConf = attachDiskBalancerConf(dn,
+ threshold, bandwidthInMB, parallelThread);
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.UPDATE, updateConf);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
private boolean shouldReturnDatanode(
HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
@@ -218,6 +335,30 @@ public class DiskBalancerManager {
}
}
+ private DiskBalancerConfiguration attachDiskBalancerConf(
+ DatanodeDetails dn, Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread) {
+ DiskBalancerConfiguration baseConf = statusMap.containsKey(dn) ?
+ statusMap.get(dn).getDiskBalancerConfiguration() :
+ new DiskBalancerConfiguration();
+ threshold.ifPresent(baseConf::setThreshold);
+ bandwidthInMB.ifPresent(baseConf::setDiskBandwidthInMB);
+ parallelThread.ifPresent(baseConf::setParallelThread);
+ return baseConf;
+ }
+
+ private void sendCommand(DatanodeDetails dn, DiskBalancerCommand command) {
+ try {
+ command.setTerm(scmContext.getTermOfLeader());
+ } catch (NotLeaderException nle) {
+ LOG.warn("Skip sending DiskBalancerCommand for Datanode {}," +
+ " since not leader SCM.", dn.getUuidString());
+ return;
+ }
+ scmNodeEventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+ new CommandForDatanode<>(dn.getUuid(), command));
+ }
+
@VisibleForTesting
public Map<DatanodeDetails, DiskBalancerStatus> getStatusMap() {
return statusMap;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 845cfc4e1a..397eed331a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -1335,21 +1335,21 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
throws IOException {
List<DatanodeAdminError> errors;
switch (request.getOpType()) {
- case start:
+ case START:
errors = impl.startDiskBalancer(
Optional.of(request.getConf().getThreshold()),
Optional.of(request.getConf().getDiskBandwidthInMB()),
Optional.of(request.getConf().getParallelThread()),
Optional.of(request.getHostsList()));
break;
- case update:
+ case UPDATE:
errors = impl.updateDiskBalancerConfiguration(
Optional.of(request.getConf().getThreshold()),
Optional.of(request.getConf().getDiskBandwidthInMB()),
Optional.of(request.getConf().getParallelThread()),
Optional.of(request.getHostsList()));
break;
- case stop:
+ case STOP:
errors = impl.stopDiskBalancer(Optional.of(request.getHostsList()));
break;
default:
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 8f2261a8d3..402cd7bc2b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1273,6 +1273,7 @@ public class SCMClientProtocolServer implements
ContainerID.valueOf(startContainerID), count, state);
}
+ @Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
int count, int clientVersion) throws IOException {
// check admin authorisation
@@ -1308,15 +1309,27 @@ public class SCMClientProtocolServer implements
public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser(), false);
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager()
+ .startDiskBalancer(threshold, bandwidthInMB, parallelThread, hosts);
}
@Override
public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser(), false);
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+ return scm.getDiskBalancerManager().stopDiskBalancer(hosts);
}
@@ -1325,8 +1338,15 @@ public class SCMClientProtocolServer implements
Optional<Double> threshold, Optional<Long> bandwidthInMB,
Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser(), false);
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().updateDiskBalancerConfiguration(
+ threshold, bandwidthInMB, parallelThread, hosts);
}
/**
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
index f58c7a1f84..fc87b145a7 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -86,4 +86,4 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
public void setAllHosts(boolean allHosts) {
this.commonOptions.setAllHosts(allHosts);
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
index 28ed4fd325..744d007539 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -84,4 +84,4 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
public void setAllHosts(boolean allHosts) {
this.commonOptions.setAllHosts(allHosts);
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 02/07: HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit b4fa3514ffb4004f322e350bbd8c2a1e16c9cc0f
Author: Symious <yi...@foxmail.com>
AuthorDate: Wed Sep 7 12:19:57 2022 +0800
HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 21 ++--
.../protocol/StorageContainerLocationProtocol.java | 17 +--
.../scm/storage/DiskBalancerConfiguration.java | 39 +++----
.../common/src/main/resources/ozone-default.xml | 8 ++
.../common/report/DiskBalancerReportPublisher.java | 66 ++++++++++++
.../common/report/ReportPublisherFactory.java | 3 +
.../common/statemachine/DatanodeStateMachine.java | 2 +
.../common/statemachine/StateContext.java | 13 +++
.../commandhandler/DiskBalancerCommandHandler.java | 117 +++++++++++++++++++++
.../states/endpoint/HeartbeatEndpointTask.java | 7 ++
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +
.../protocol/commands/DiskBalancerCommand.java | 74 +++++++++++++
.../common/report/TestReportPublisher.java | 42 ++++++++
.../common/report/TestReportPublisherFactory.java | 12 +++
...inerLocationProtocolClientSideTranslatorPB.java | 81 ++++++++++++--
.../src/main/proto/ScmAdminProtocol.proto | 20 ++++
.../interface-client/src/main/proto/hdds.proto | 10 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 17 +++
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 9 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 109 +++++++++++++------
.../hdds/scm/node/DiskBalancerReportHandler.java | 65 ++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 2 -
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 17 +++
...inerLocationProtocolServerSideTranslatorPB.java | 51 ++++++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 22 ++--
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 23 ++++
.../hdds/scm/server/StorageContainerManager.java | 5 +
.../hadoop/hdds/scm/container/MockNodeManager.java | 51 ++++++++-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../hdds/scm/node/TestDiskBalancerManager.java | 34 ++++++
.../testutils/ReplicationNodeManagerMock.java | 6 ++
.../hdds/scm/cli/ContainerOperationClient.java | 29 ++---
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +
35 files changed, 902 insertions(+), 96 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 787f023df2..899c3acf44 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -46,6 +46,10 @@ public final class HddsConfigKeys {
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
"60s";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL =
+ "hdds.disk.balancer.report.interval";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT =
+ "60s";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 8f09b43424..b181e9ee8f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -454,7 +454,8 @@ public interface ScmClient extends Closeable {
String scmId) throws IOException;
/**
- * Get DiskBalancer status.
+ * Get DiskBalancer report.
+ * REPORT shows the current volume density of datanodes.
* @param count top datanodes that need balancing
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
@@ -464,33 +465,39 @@ public interface ScmClient extends Closeable {
/**
* Get DiskBalancer status.
+ * STATUS shows the running status of DiskBalancer on datanodes.
* @param hosts If hosts is not null, return status of hosts; If hosts is
* null, return status of all datanodes in balancing.
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidth,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 1d7e83b247..8accd5b9ef 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -482,26 +482,31 @@ public interface StorageContainerLocationProtocol extends Closeable {
* Get DiskBalancer status.
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus,
+ int clientVersion) throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index 54709e4c17..e460e0b6eb 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,7 +36,7 @@ public final class DiskBalancerConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerConfiguration.class);
- @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ @Config(key = "volume.density.threshold", type = ConfigType.DOUBLE,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "Threshold is a percentage in the range of 0 to 100. A " +
"datanode is considered balanced if for each volume, the " +
@@ -45,12 +45,12 @@ public final class DiskBalancerConfiguration {
" of the entire datanode) no more than the threshold.")
private double threshold = 10d;
- @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.LONG,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "The max balance speed.")
- private double diskBandwidth = 10;
+ private long diskBandwidthInMB = 10;
- @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ @Config(key = "parallel.thread", type = ConfigType.INT,
defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
description = "The max parallel balance thread count.")
private int parallelThread = 5;
@@ -86,21 +86,21 @@ public final class DiskBalancerConfiguration {
*
* @return max disk bandwidth per second
*/
- public double getDiskBandwidth() {
- return diskBandwidth;
+ public double getDiskBandwidthInMB() {
+ return diskBandwidthInMB;
}
/**
* Sets the disk bandwidth value for Disk Balancer.
*
- * @param diskBandwidth the bandwidth to control balance speed
+ * @param diskBandwidthInMB the bandwidth to control balance speed
*/
- public void setDiskBandwidth(double diskBandwidth) {
- if (diskBandwidth <= 0d) {
+ public void setDiskBandwidthInMB(long diskBandwidthInMB) {
+ if (diskBandwidthInMB <= 0L) {
throw new IllegalArgumentException(
- "diskBandwidth must be a value larger than 0.");
+ "diskBandwidthInMB must be a value larger than 0.");
}
- this.diskBandwidth = diskBandwidth;
+ this.diskBandwidthInMB = diskBandwidthInMB;
}
/**
@@ -124,6 +124,7 @@ public final class DiskBalancerConfiguration {
}
this.parallelThread = parallelThread;
}
+
@Override
public String toString() {
return String.format("Disk Balancer Configuration values:%n" +
@@ -132,7 +133,7 @@ public final class DiskBalancerConfiguration {
"%-50s %s%n" +
"%-50s %s%n",
"Key", "Value",
- "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidthInMB,
"Parallel Thread", parallelThread);
}
@@ -141,21 +142,21 @@ public final class DiskBalancerConfiguration {
HddsProtos.DiskBalancerConfigurationProto.newBuilder();
builder.setThreshold(threshold)
- .setDiskBandwidth(diskBandwidth)
+ .setDiskBandwidthInMB(diskBandwidthInMB)
.setParallelThread(parallelThread);
return builder;
}
- static DiskBalancerConfiguration fromProtobuf(
+ public static DiskBalancerConfiguration fromProtobuf(
@Nonnull HddsProtos.DiskBalancerConfigurationProto proto,
- @Nonnull OzoneConfiguration ozoneConfiguration) {
+ @Nonnull ConfigurationSource configurationSource) {
DiskBalancerConfiguration config =
- ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ configurationSource.getObject(DiskBalancerConfiguration.class);
if (proto.hasThreshold()) {
config.setThreshold(proto.getThreshold());
}
- if (proto.hasDiskBandwidth()) {
- config.setDiskBandwidth(proto.getDiskBandwidth());
+ if (proto.hasDiskBandwidthInMB()) {
+ config.setDiskBandwidthInMB(proto.getDiskBandwidthInMB());
}
if (proto.hasParallelThread()) {
config.setParallelThread(proto.getParallelThread());
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c166e534d5..741d078cec 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -381,6 +381,14 @@
datanode periodically send pipeline report to SCM. Unit could be
defined with postfix (ns,ms,s,m,h,d)</description>
</property>
+ <property>
+ <name>hdds.disk.balancer.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, CONTAINER, DISK_BALANCER</tag>
+ <description>Time interval of the datanode to send disk balancer report. Each
+ datanode periodically sends disk balancer report to SCM. Unit could be
+ defined with postfix (ns,ms,s,m,h,d)</description>
+ </property>
<property>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
new file mode 100644
index 0000000000..2bb78c1ee6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.report;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT;
+
+
+/**
+ * Publishes DiskBalancer report which will be sent to SCM as part of heartbeat.
+ * DiskBalancer Report consist of the following information:
+ * - isBalancerRunning
+ * - balancedBytes
+ * - DiskBalancerConfiguration
+ */
+public class DiskBalancerReportPublisher extends
+ ReportPublisher<DiskBalancerReportProto> {
+
+ private Long diskBalancerReportInterval = null;
+
+ @Override
+ protected long getReportFrequency() {
+ if (diskBalancerReportInterval == null) {
+ diskBalancerReportInterval = getConf().getTimeDuration(
+ HDDS_DISK_BALANCER_REPORT_INTERVAL,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+ getConf());
+
+ Preconditions.checkState(
+ heartbeatFrequency <= diskBalancerReportInterval,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL +
+ " cannot be configured lower than heartbeat frequency " +
+ heartbeatFrequency + ".");
+ }
+ return diskBalancerReportInterval;
+ }
+
+ @Override
+ protected DiskBalancerReportProto getReport() {
+ return getContext().getParent().getContainer().getDiskBalancerReport();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index 3be1b5e077..19d2806919 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -55,6 +56,8 @@ public class ReportPublisherFactory {
report2publisher.put(PipelineReportsProto.class,
PipelineReportPublisher.class);
report2publisher.put(CRLStatusReport.class, CRLStatusReportPublisher.class);
+ report2publisher.put(DiskBalancerReportProto.class,
+ DiskBalancerReportPublisher.class);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 9677144054..1623c1ae28 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.Comm
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CreatePipelineCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DiskBalancerCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.FinalizeNewLayoutVersionCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ReconstructECContainersCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.RefreshVolumeUsageCommandHandler;
@@ -258,6 +259,7 @@ public class DatanodeStateMachine implements Closeable {
supervisor::nodeStateUpdated))
.addHandler(new FinalizeNewLayoutVersionCommandHandler())
.addHandler(new RefreshVolumeUsageCommandHandler())
+ .addHandler(new DiskBalancerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 6bbf8e4794..70fafa2c84 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
@@ -98,6 +99,9 @@ public class StateContext {
@VisibleForTesting
static final String CRL_STATUS_REPORT_PROTO_NAME =
CRLStatusReport.getDescriptor().getFullName();
+ @VisibleForTesting
+ static final String DISK_BALANCER_REPORT_PROTO_NAME =
+ DiskBalancerReportProto.getDescriptor().getFullName();
static final Logger LOG =
LoggerFactory.getLogger(StateContext.class);
@@ -113,6 +117,7 @@ public class StateContext {
private final AtomicReference<Message> nodeReport;
private final AtomicReference<Message> pipelineReports;
private final AtomicReference<Message> crlStatusReport;
+ private final AtomicReference<Message> diskBalancerReport;
// Incremental reports are queued in the map below
private final Map<InetSocketAddress, List<Message>>
incrementalReportsQueue;
@@ -180,6 +185,7 @@ public class StateContext {
nodeReport = new AtomicReference<>();
pipelineReports = new AtomicReference<>();
crlStatusReport = new AtomicReference<>(); // Certificate Revocation List
+ diskBalancerReport = new AtomicReference<>();
endpoints = new HashSet<>();
containerActions = new HashMap<>();
pipelineActions = new HashMap<>();
@@ -206,6 +212,8 @@ public class StateContext {
type2Reports.put(PIPELINE_REPORTS_PROTO_NAME, pipelineReports);
fullReportTypeList.add(CRL_STATUS_REPORT_PROTO_NAME);
type2Reports.put(CRL_STATUS_REPORT_PROTO_NAME, crlStatusReport);
+ fullReportTypeList.add(DISK_BALANCER_REPORT_PROTO_NAME);
+ type2Reports.put(DISK_BALANCER_REPORT_PROTO_NAME, diskBalancerReport);
}
/**
@@ -958,6 +966,11 @@ public class StateContext {
return crlStatusReport.get();
}
+ @VisibleForTesting
+ public Message getDiskBalancerReport() {
+ return diskBalancerReport.get();
+ }
+
public void configureReconHeartbeatFrequency() {
reconHeartbeatFrequency.set(getReconHeartbeatInterval(conf));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
new file mode 100644
index 0000000000..37f275dbb4
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Handler for DiskBalancer command received from SCM.
+ */
+public class DiskBalancerCommandHandler implements CommandHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerCommandHandler.class);
+
+ private AtomicLong invocationCount = new AtomicLong(0);
+ private long totalTime;
+
+ /**
+ * Constructs a diskBalancerCommand handler.
+ */
+ public DiskBalancerCommandHandler() {
+ }
+
+ /**
+ * Handles a given SCM command.
+ *
+ * @param command - SCM Command
+ * @param ozoneContainer - Ozone Container.
+ * @param context - Current Context.
+ * @param connectionManager - The SCMs that we are talking to.
+ */
+ @Override
+ public void handle(SCMCommand command, OzoneContainer ozoneContainer,
+ StateContext context, SCMConnectionManager connectionManager) {
+ invocationCount.incrementAndGet();
+ // TODO: Do start/stop/update operation
+ }
+
+ /**
+ * Returns the command type that this command handler handles.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getCommandType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ /**
+ * Returns number of times this handler has been invoked.
+ *
+ * @return int
+ */
+ @Override
+ public int getInvocationCount() {
+ return (int)invocationCount.get();
+ }
+
+ /**
+ * Returns the average time this function takes to run.
+ *
+ * @return long
+ */
+ @Override
+ public long getAverageRunTime() {
+ if (invocationCount.get() > 0) {
+ return totalTime / invocationCount.get();
+ }
+ return 0;
+ }
+
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
+ @Override
+ public int getQueuedCount() {
+ return 0;
+ }
+
+ private void startDiskBalancer(DiskBalancerConfiguration configuration) {
+ // Todo: add implementation to start DiskBalancer
+ }
+
+ private void stopDiskBalancer() {
+ // Todo: add implementation to stop DiskBalancer
+ }
+
+ private void updateDiskBalancer(DiskBalancerConfiguration
+ configuration) {
+ // Todo: add implementation to update diskBalancer configuration
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 44f0eae49e..0685650361 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
import org.apache.hadoop.ozone.protocol.commands.FinalizeNewLayoutVersionCommand;
import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
import org.apache.hadoop.ozone.protocol.commands.RefreshVolumeUsageCommand;
@@ -416,6 +417,12 @@ public class HeartbeatEndpointTask
commandResponseProto.getRefreshVolumeUsageCommandProto());
processCommonCommand(commandResponseProto, refreshVolumeUsageCommand);
break;
+ case diskBalancerCommand:
+ DiskBalancerCommand diskBalancerCommand =
+ DiskBalancerCommand.getFromProtobuf(
+ commandResponseProto.getDiskBalancerCommandProto(), conf);
+ processCommonCommand(commandResponseProto, diskBalancerCommand);
+ break;
default:
throw new IllegalArgumentException("Unknown response : "
+ commandResponseProto.getCommandType().name());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index aef3965dcd..f81b58e317 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.SecurityConfig;
@@ -590,4 +591,8 @@ public class OzoneContainer {
return replicationServer;
}
+ public DiskBalancerReportProto getDiskBalancerReport() {
+ // TODO: Return real disk balancer report
+ return null;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
new file mode 100644
index 0000000000..ea97f34ca0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerCommandProto;
+
+/**
+ * Informs a datanode to update DiskBalancer status.
+ */
+public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
+
+ private final boolean shouldRun;
+ private final DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public DiskBalancerCommand(final boolean shouldRun,
+ final DiskBalancerConfiguration diskBalancerConfiguration) {
+ this.shouldRun = shouldRun;
+ this.diskBalancerConfiguration = diskBalancerConfiguration;
+ }
+
+ /**
+ * Returns the type of this command.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ @Override
+ public DiskBalancerCommandProto getProto() {
+ return DiskBalancerCommandProto.newBuilder()
+ .setShouldRun(shouldRun)
+ .setDiskBalancerConf(diskBalancerConfiguration.toProtobufBuilder())
+ .build();
+ }
+
+ public static DiskBalancerCommand getFromProtobuf(DiskBalancerCommandProto
+ diskbalancerCommandProto, ConfigurationSource configuration) {
+ Preconditions.checkNotNull(diskbalancerCommandProto);
+ return new DiskBalancerCommand(diskbalancerCommandProto.getShouldRun(),
+ DiskBalancerConfiguration.fromProtobuf(
+ diskbalancerCommandProto.getDiskBalancerConf(), configuration));
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index 60404fa36b..2f38e87fed 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -31,7 +31,9 @@ import org.apache.hadoop.hdds.HddsIdFactory;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
import org.apache.hadoop.hdds.protocol.proto.
@@ -39,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto.
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.junit.jupiter.api.BeforeAll;
@@ -217,6 +220,45 @@ public class TestReportPublisher {
executorService.shutdown();
}
+ @Test
+ public void testDiskBalancerReportPublisher() throws IOException {
+ StateContext dummyContext = mock(StateContext.class);
+ DatanodeStateMachine dummyStateMachine =
+ mock(DatanodeStateMachine.class);
+ OzoneContainer dummyContainer = mock(OzoneContainer.class);
+
+ DiskBalancerReportProto.Builder builder =
+ DiskBalancerReportProto.newBuilder();
+ builder.setIsRunning(true);
+ builder.setBalancedBytes(1L);
+ builder.setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build());
+ DiskBalancerReportProto dummyReport = builder.build();
+
+ ReportPublisher publisher = new DiskBalancerReportPublisher();
+ when(dummyContext.getParent()).thenReturn(dummyStateMachine);
+ when(dummyStateMachine.getContainer()).thenReturn(dummyContainer);
+ when(dummyContainer.getDiskBalancerReport()).thenReturn(dummyReport);
+ publisher.setConf(config);
+
+ ScheduledExecutorService executorService = HadoopExecutors
+ .newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("Unit test ReportManager Thread - %d").build());
+ publisher.init(dummyContext, executorService);
+ Message report =
+ ((DiskBalancerReportPublisher) publisher).getReport();
+ assertNotNull(report);
+ for (Descriptors.FieldDescriptor descriptor :
+ report.getDescriptorForType().getFields()) {
+ if (descriptor.getNumber() ==
+ DiskBalancerReportProto.ISRUNNING_FIELD_NUMBER) {
+ assertEquals(true, report.getField(descriptor));
+ }
+ }
+ executorService.shutdown();
+ }
+
/**
* Get a datanode details.
*
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
index 44e1389bf7..1ebb05218d 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.junit.jupiter.api.Test;
@@ -66,6 +67,17 @@ public class TestReportPublisherFactory {
assertEquals(conf, publisher.getConf());
}
+ @Test
+ public void testGetDiskBalancerReportPublisher() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ ReportPublisherFactory factory = new ReportPublisherFactory(conf);
+ ReportPublisher publisher = factory
+ .getPublisherFor(DiskBalancerReportProto.class);
+ assertEquals(DiskBalancerReportPublisher.class,
+ publisher.getClass());
+ assertEquals(conf, publisher.getConf());
+ }
+
@Test
public void testInvalidReportPublisher() {
OzoneConfiguration conf = new OzoneConfiguration();
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 62d80e3a65..7a341820e9 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -50,6 +50,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1133,11 +1136,14 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
DatanodeDiskBalancerInfoRequestProto.newBuilder()
.setInfoType(DatanodeDiskBalancerInfoType.status);
hosts.ifPresent(requestBuilder::addAllHosts);
+ status.ifPresent(requestBuilder::setStatus);
DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
DatanodeDiskBalancerInfoResponseProto response =
@@ -1149,22 +1155,83 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.start)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.stop);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.update)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 649d61af9f..807e2634b6 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -84,6 +84,7 @@ message ScmContainerLocationRequest {
optional SingleNodeQueryRequestProto singleNodeQueryRequest = 45;
optional GetContainersOnDecomNodeRequestProto getContainersOnDecomNodeRequest = 46;
optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 47;
+ optional DatanodeDiskBalancerOpRequestProto datanodeDiskBalancerOpRequest = 48;
}
message ScmContainerLocationResponse {
@@ -139,6 +140,7 @@ message ScmContainerLocationResponse {
optional SingleNodeQueryResponseProto singleNodeQueryResponse = 45;
optional GetContainersOnDecomNodeResponseProto getContainersOnDecomNodeResponse = 46;
optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 47;
+ optional DatanodeDiskBalancerOpResponseProto datanodeDiskBalancerOpResponse = 48;
enum Status {
OK = 1;
@@ -193,6 +195,7 @@ enum Type {
SingleNodeQuery = 41;
GetContainersOnDecomNode = 42;
DatanodeDiskBalancerInfo = 43;
+ DatanodeDiskBalancerOp = 44;
}
/**
@@ -369,12 +372,29 @@ message DatanodeDiskBalancerInfoRequestProto {
required DatanodeDiskBalancerInfoType infoType = 1;
optional uint32 count = 2;
repeated string hosts = 3;
+ optional DiskBalancerRunningStatus status = 4;
}
message DatanodeDiskBalancerInfoResponseProto {
repeated DatanodeDiskBalancerInfoProto info = 1;
}
+enum DatanodeDiskBalancerOpType{
+ start = 1;
+ stop = 2;
+ update = 3;
+}
+
+message DatanodeDiskBalancerOpRequestProto {
+ required DatanodeDiskBalancerOpType opType = 1;
+ repeated string hosts = 2;
+ optional DiskBalancerConfigurationProto conf = 3;
+}
+
+message DatanodeDiskBalancerOpResponseProto {
+ repeated DatanodeAdminErrorResponseProto failedHosts = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 424686333a..6109ac93fe 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -500,13 +500,19 @@ message CompactionLogEntryProto {
message DiskBalancerConfigurationProto {
optional double threshold = 1;
- optional double diskBandwidth = 2;
+ optional uint64 diskBandwidthInMB = 2;
optional int32 parallelThread = 3;
}
+enum DiskBalancerRunningStatus {
+ RUNNING = 1;
+ STOPPED = 2;
+ UNKNOWN = 3;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
- optional bool diskBalancerRunning = 3;
+ optional DiskBalancerRunningStatus runningStatus = 3;
optional DiskBalancerConfigurationProto diskBalancerConf = 4;
}
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 2994073c02..7a265dd0ba 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -139,6 +139,7 @@ message SCMHeartbeatRequestProto {
optional PipelineReportsProto pipelineReports = 8;
optional LayoutVersionProto dataNodeLayoutVersion = 9;
optional CommandQueueReportProto commandQueueReport = 10;
+ optional DiskBalancerReportProto diskBalancerReport = 11;
}
message CommandQueueReportProto {
@@ -328,6 +329,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommand = 9;
refreshVolumeUsageInfo = 10;
reconstructECContainersCommand = 11;
+ diskBalancerCommand = 12;
}
// TODO: once we start using protoc 3.x, refactor this message using "oneof"
required Type commandType = 1;
@@ -343,6 +345,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommandProto = 10;
optional RefreshVolumeUsageCommandProto refreshVolumeUsageCommandProto = 11;
optional ReconstructECContainersCommandProto reconstructECContainersCommandProto = 12;
+ optional DiskBalancerCommandProto diskBalancerCommandProto = 13;
// If running upon Ratis, holds term of underlying RaftServer iff current
@@ -445,6 +448,14 @@ message DatanodeDetailsAndReplicaIndexProto {
required int32 replicaIndex = 2;
}
+/**
+This command asks the datanode to update diskBalancer status
+ */
+message DiskBalancerCommandProto {
+ required bool shouldRun = 1;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 2;
+}
+
/**
This command asks the datanode to create a pipeline.
*/
@@ -483,6 +494,12 @@ message CRLStatusReport {
repeated int64 pendingCrlIds=2;
}
+message DiskBalancerReportProto {
+ required bool isRunning = 1;
+ optional uint64 balancedBytes = 2;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 3;
+}
+
/**
* This command asks the datanode to process a new CRL.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 0cc205b2ff..d22a878249 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatu
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
@@ -217,6 +218,14 @@ public final class SCMEvents {
new TypedEvent<>(CRLStatusReportFromDatanode.class,
"Crl_Status_Report");
+ /**
+ * DiskBalancer reports are send out by Datanodes. This report is received by
+ * SCMDatanodeHeartbeatDispatcher and DiskBalancer_Report Event is generated.
+ */
+ public static final TypedEvent<DiskBalancerReportFromDatanode>
+ DISK_BALANCER_REPORT = new TypedEvent<>(
+ DiskBalancerReportFromDatanode.class, "DiskBalancer_Report");
+
/**
* Private Ctor. Never Constructed.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index 0b16c1aa95..c6114c52f5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -22,7 +22,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -47,10 +48,13 @@ public class DiskBalancerManager {
public static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private OzoneConfiguration conf;
private final EventPublisher scmNodeEventPublisher;
private final SCMContext scmContext;
private final NodeManager nodeManager;
private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private Map<DatanodeDetails, Long> balancedBytesMap;
private boolean useHostnames;
/**
@@ -60,6 +64,7 @@ public class DiskBalancerManager {
EventPublisher eventPublisher,
SCMContext scmContext,
NodeManager nodeManager) {
+ this.conf = conf;
this.scmNodeEventPublisher = eventPublisher;
this.scmContext = scmContext;
this.nodeManager = nodeManager;
@@ -95,42 +100,60 @@ public class DiskBalancerManager {
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus = status.orElse(null);
+
+ if (filterDns != null) {
+ return filterDns.stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(nodeManager::getDatanodeInfo)
+ .map(dn -> getInfoProto(dn, clientVersion))
+ .collect(Collectors.toList());
+ } else {
+ return nodeManager.getAllNodes().stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(dn -> getInfoProto((DatanodeInfo)dn, clientVersion))
+ .collect(Collectors.toList());
}
- return statusList;
}
- private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ private boolean shouldReturnDatanode(
+ HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
- if (hosts == null || hosts.isEmpty()) {
- return isRunning(datanodeDetails);
- } else {
- return hosts.contains(datanodeDetails);
+ boolean shouldReturn = true;
+ // If status specified, do not return if status not match.
+ if (status != null && getRunningStatus(datanodeDetails) != status) {
+ shouldReturn = false;
}
+ return shouldReturn;
}
+ private HddsProtos.DatanodeDiskBalancerInfoProto getInfoProto(
+ DatanodeInfo dn, int clientVersion) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(dn);
+ HddsProtos.DiskBalancerRunningStatus runningStatus =
+ getRunningStatus(dn);
+ HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
+ HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(dn.toProto(clientVersion))
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setRunningStatus(getRunningStatus(dn));
+ if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ builder.setDiskBalancerConf(statusMap.get(dn)
+ .getDiskBalancerConfiguration().toProtobufBuilder());
+ }
+ return builder.build();
+ }
/**
* Get volume density for a specific DatanodeDetails node.
*
@@ -144,8 +167,7 @@ public class DiskBalancerManager {
DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
double totalCapacity = 0d, totalUsed = 0d;
- for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
- datanodeInfo.getStorageReports()) {
+ for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
totalCapacity += reportProto.getCapacity();
totalUsed += reportProto.getScmUsed();
}
@@ -162,10 +184,17 @@ public class DiskBalancerManager {
return volumeDensitySum;
}
- private boolean isRunning(DatanodeDetails datanodeDetails) {
- return statusMap
- .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
- .isRunning();
+ private HddsProtos.DiskBalancerRunningStatus getRunningStatus(
+ DatanodeDetails datanodeDetails) {
+ if (!statusMap.containsKey(datanodeDetails)) {
+ return HddsProtos.DiskBalancerRunningStatus.UNKNOWN;
+ } else {
+ if (statusMap.get(datanodeDetails).isRunning()) {
+ return HddsProtos.DiskBalancerRunningStatus.RUNNING;
+ } else {
+ return HddsProtos.DiskBalancerRunningStatus.STOPPED;
+ }
+ }
}
@VisibleForTesting
@@ -173,4 +202,24 @@ public class DiskBalancerManager {
statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
new DiskBalancerConfiguration()));
}
+
+ public void processDiskBalancerReport(DiskBalancerReportProto reportProto,
+ DatanodeDetails dn) {
+ boolean isRunning = reportProto.getIsRunning();
+ DiskBalancerConfiguration diskBalancerConfiguration =
+ reportProto.hasDiskBalancerConf() ?
+ DiskBalancerConfiguration.fromProtobuf(
+ reportProto.getDiskBalancerConf(), conf) :
+ new DiskBalancerConfiguration();
+ statusMap.put(dn, new DiskBalancerStatus(isRunning,
+ diskBalancerConfiguration));
+ if (reportProto.hasBalancedBytes()) {
+ balancedBytesMap.put(dn, reportProto.getBalancedBytes());
+ }
+ }
+
+ @VisibleForTesting
+ public Map<DatanodeDetails, DiskBalancerStatus> getStatusMap() {
+ return statusMap;
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
new file mode 100644
index 0000000000..47158972ec
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles DiskBalancer Reports from datanode.
+ */
+public class DiskBalancerReportHandler implements
+ EventHandler<DiskBalancerReportFromDatanode> {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(DiskBalancerReportHandler.class);
+
+ private DiskBalancerManager diskBalancerManager;
+
+ public DiskBalancerReportHandler(DiskBalancerManager diskBalancerManager) {
+ this.diskBalancerManager = diskBalancerManager;
+ }
+
+ @Override
+ public void onMessage(DiskBalancerReportFromDatanode reportFromDatanode,
+ EventPublisher publisher) {
+ Preconditions.checkNotNull(reportFromDatanode);
+ DatanodeDetails dn = reportFromDatanode.getDatanodeDetails();
+ DiskBalancerReportProto diskBalancerReportProto =
+ reportFromDatanode.getReport();
+ Preconditions.checkNotNull(dn,
+ "DiskBalancer Report is missing DatanodeDetails.");
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("Processing diskBalancer report for dn: {}", dn);
+ }
+ try {
+ diskBalancerManager.processDiskBalancerReport(
+ diskBalancerReportProto, dn);
+ } catch (Exception e) {
+ LOGGER.error("Failed to process diskBalancer report={} from dn={}.",
+ diskBalancerReportProto, dn, e);
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
index a064a6ff9f..8d2cb26c36 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -32,8 +32,6 @@ public class DiskBalancerStatus {
private boolean isRunning;
private DiskBalancerConfiguration diskBalancerConfiguration;
- public static final DiskBalancerStatus DUMMY_STATUS =
- new DiskBalancerStatus(false, new DiskBalancerConfiguration());
public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
this.isRunning = isRunning;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 21bcd1f78a..6063e6dd88 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -175,6 +175,14 @@ public interface NodeManager extends StorageContainerNodeProtocol,
*/
DatanodeUsageInfo getUsageInfo(DatanodeDetails dn);
+ /**
+ * Get the datanode info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeInfo of the specified datanode
+ */
+ DatanodeInfo getDatanodeInfo(DatanodeDetails dn);
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails DatanodeDetails.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 152c8fd659..f4d5fc0246 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -983,6 +983,23 @@ public class SCMNodeManager implements NodeManager {
return usageInfo;
}
+ /**
+ * Get the usage info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeUsageInfo of the specified datanode
+ */
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ try {
+ return nodeStateManager.getNode(dn);
+ } catch (NodeNotFoundException e) {
+ LOG.warn("Cannot retrieve DatanodeInfo, datanode {} not found.",
+ dn.getUuid());
+ return null;
+ }
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index aa0a197dc1..845cfc4e1a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -723,6 +724,13 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
request.getDatanodeDiskBalancerInfoRequest(),
request.getVersion()))
.build();
+ case DatanodeDiskBalancerOp:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerOpResponse(getDatanodeDiskBalancerOp(
+ request.getDatanodeDiskBalancerOpRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1309,7 +1317,9 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
break;
case status:
infoProtoList = impl.getDiskBalancerStatus(
- Optional.of(request.getHostsList()), clientVersion);
+ Optional.of(request.getHostsList()),
+ Optional.of(request.getStatus()),
+ clientVersion);
break;
default:
infoProtoList = null;
@@ -1318,4 +1328,43 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
.addAllInfo(infoProtoList)
.build();
}
+
+ public DatanodeDiskBalancerOpResponseProto getDatanodeDiskBalancerOp(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerOpRequestProto request)
+ throws IOException {
+ List<DatanodeAdminError> errors;
+ switch (request.getOpType()) {
+ case start:
+ errors = impl.startDiskBalancer(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case update:
+ errors = impl.updateDiskBalancerConfiguration(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case stop:
+ errors = impl.stopDiskBalancer(Optional.of(request.getHostsList()));
+ break;
+ default:
+ errors = new ArrayList<>();
+ }
+
+ DatanodeDiskBalancerOpResponseProto.Builder response =
+ DatanodeDiskBalancerOpResponseProto.newBuilder();
+ for (DatanodeAdminError e : errors) {
+ DatanodeAdminErrorResponseProto.Builder error =
+ DatanodeAdminErrorResponseProto.newBuilder();
+ error.setHost(e.getHostname());
+ error.setError(e.getError());
+ response.addFailedHosts(error);
+ }
+ return response.build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index dacdbe0e3f..79cb469047 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1289,7 +1289,9 @@ public class SCMClientProtocolServer implements
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
// check admin authorisation
try {
getScm().checkAdminAccess(getRemoteUser(), true);
@@ -1298,29 +1300,33 @@ public class SCMClientProtocolServer implements
throw e;
}
- return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts, status,
clientVersion);
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index b6dc6f599b..d670be9dd0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
@@ -57,6 +58,7 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DISK_BALANCER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents
.INCREMENTAL_CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NODE_REPORT;
@@ -201,6 +203,15 @@ public final class SCMDatanodeHeartbeatDispatcher {
commandStatusReport));
}
}
+
+ if (heartbeat.hasDiskBalancerReport()) {
+ LOG.debug("Dispatching DiskBalancer Report.");
+ eventPublisher.fireEvent(
+ DISK_BALANCER_REPORT,
+ new DiskBalancerReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getDiskBalancerReport()));
+ }
}
return commands;
@@ -447,4 +458,16 @@ public final class SCMDatanodeHeartbeatDispatcher {
super(datanodeDetails, report);
}
}
+
+ /**
+ * DiskBalancer report event payload with origin.
+ */
+ public static class DiskBalancerReportFromDatanode
+ extends ReportFromDatanode<DiskBalancerReportProto> {
+
+ public DiskBalancerReportFromDatanode(DatanodeDetails datanodeDetails,
+ DiskBalancerReportProto report) {
+ super(datanodeDetails, report);
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 3e55d7ded0..518f7d2d6b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerReportHandler;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.security.SecretKeyManagerService;
import org.apache.hadoop.hdds.scm.security.RootCARotationManager;
@@ -508,6 +509,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PipelineActionHandler(pipelineManager, scmContext, configuration);
CRLStatusReportHandler crlStatusReportHandler =
new CRLStatusReportHandler(certificateStore, configuration);
+ DiskBalancerReportHandler diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
@@ -585,6 +588,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmNodeManager.registerSendCommandNotify(
SCMCommandProto.Type.deleteBlocksCommand,
scmBlockManager.getDeletedBlockLog()::onSent);
+ eventQueue.addHandler(SCMEvents.DISK_BALANCER_REPORT,
+ diskBalancerReportHandler);
}
private void initializeCertificateClient() throws IOException {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 84f3684ab7..df911462fa 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -350,7 +350,36 @@ public class MockNodeManager implements NodeManager {
*/
@Override
public List<DatanodeDetails> getAllNodes() {
- return new ArrayList<>(nodeMetricMap.keySet());
+ // mock storage reports for TestDiskBalancer
+ List<DatanodeDetails> healthyNodesWithInfo = new ArrayList<>();
+ for (Map.Entry<DatanodeDetails, SCMNodeStat> entry:
+ nodeMetricMap.entrySet()) {
+ NodeStatus nodeStatus = NodeStatus.inServiceHealthy();
+ if (staleNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceStale();
+ } else if (deadNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceDead();
+ }
+ DatanodeInfo di = new DatanodeInfo(entry.getKey(), nodeStatus,
+ UpgradeUtils.defaultLayoutVersionProto());
+
+ long capacity = entry.getValue().getCapacity().get();
+ long used = entry.getValue().getScmUsed().get();
+ long remaining = entry.getValue().getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+
+ healthyNodesWithInfo.add(di);
+ }
+ return healthyNodesWithInfo;
}
/**
@@ -413,6 +442,26 @@ public class MockNodeManager implements NodeManager {
return new DatanodeUsageInfo(datanodeDetails, stat);
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dd) {
+ DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(),
+ UpgradeUtils.defaultLayoutVersionProto());
+ long capacity = nodeMetricMap.get(dd).getCapacity().get();
+ long used = nodeMetricMap.get(dd).getScmUsed().get();
+ long remaining = nodeMetricMap.get(dd).getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+ return di;
+ }
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails - datanode details.
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
index 9649159de3..0838e56ba8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
@@ -245,6 +245,11 @@ public class SimpleMockNodeManager implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
return null;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
index 541c9764b7..7005dea292 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -21,8 +21,10 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.ozone.test.GenericTestUtils;
@@ -33,6 +35,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
+import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class TestDiskBalancerManager {
private NodeManager nodeManager;
private OzoneConfiguration conf;
private String storageDir;
+ private DiskBalancerReportHandler diskBalancerReportHandler;
+ private Random random;
@BeforeEach
public void setup() throws Exception {
@@ -56,6 +61,9 @@ public class TestDiskBalancerManager {
nodeManager = new MockNodeManager(true, 3);
diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
SCMContext.emptyContext(), nodeManager);
+ diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
+ random = new Random();
}
@Test
@@ -73,6 +81,7 @@ public class TestDiskBalancerManager {
@Test
public void testDatanodeDiskBalancerStatus() throws IOException {
diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(1));
// Simulate users asking all status of 3 datanodes
List<String> dns = nodeManager.getAllNodes().stream().map(
@@ -81,6 +90,7 @@ public class TestDiskBalancerManager {
List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(3, statusProtoList.size());
@@ -92,8 +102,32 @@ public class TestDiskBalancerManager {
statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(1, statusProtoList.size());
}
+
+ @Test
+ public void testHandleDiskBalancerReportFromDatanode() {
+ for (DatanodeDetails dn: nodeManager.getAllNodes()) {
+ diskBalancerReportHandler.onMessage(
+ new DiskBalancerReportFromDatanode(dn, generateRandomReport()), null);
+ }
+
+ Assertions.assertEquals(3, diskBalancerManager.getStatusMap().size());
+ }
+
+ private DiskBalancerReportProto generateRandomReport() {
+ return DiskBalancerReportProto.newBuilder()
+ .setIsRunning(random.nextBoolean())
+ .setBalancedBytes(random.nextInt(10000))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(random.nextInt(99))
+ .setParallelThread(random.nextInt(4) + 1)
+ .setDiskBandwidthInMB(random.nextInt(99) + 1)
+ .build())
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 92a6fd455d..4b83abff83 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -195,6 +196,11 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index b16b4dc18a..fd20509bae 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -571,31 +571,34 @@ public class ContainerOperationClient implements ScmClient {
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
- storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
- hosts);
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.startDiskBalancer(threshold,
+ bandwidthInMB, parallelThread, hosts);
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.stopDiskBalancer(hosts);
+ return storageContainerLocationClient.stopDiskBalancer(hosts);
}
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException {
return storageContainerLocationClient.getDiskBalancerStatus(hosts,
- ClientVersion.CURRENT_VERSION);
+ runningStatus, ClientVersion.CURRENT_VERSION);
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidth,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
- bandwidth, hosts);
+ return storageContainerLocationClient.updateDiskBalancerConfiguration(
+ threshold, bandwidth, parallelThread, hosts);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
index c047cd3ee0..7eaca03fed 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -48,6 +49,7 @@ public class TestDiskBalancer {
private static ScmClient storageClient;
private static MiniOzoneCluster cluster;
private static OzoneConfiguration ozoneConf;
+ private static DiskBalancerManager diskBalancerManager;
@BeforeAll
public static void setup() throws Exception {
@@ -57,6 +59,8 @@ public class TestDiskBalancer {
cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
storageClient = new ContainerOperationClient(ozoneConf);
cluster.waitForClusterToBeReady();
+ diskBalancerManager = cluster.getStorageContainerManager()
+ .getDiskBalancerManager();
for (DatanodeDetails dn: cluster.getStorageContainerManager()
.getScmNodeManager().getAllNodes()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 03/07: HDDS-7205. DiskBalancer CLI (#3739)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4e20553120220147b402afde8cf928fb76c2e15c
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 15 10:02:49 2022 +0800
HDDS-7205. DiskBalancer CLI (#3739)
---
.../hdds/scm/server/SCMClientProtocolServer.java | 6 +-
.../hdds/scm/cli/datanode/DatanodeCommands.java | 3 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 99 ++++++++
.../cli/datanode/DiskBalancerReportSubcommand.java | 70 ++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 100 ++++++++
.../cli/datanode/DiskBalancerStatusSubcommand.java | 103 ++++++++
.../cli/datanode/DiskBalancerStopSubcommand.java | 82 +++++++
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 98 ++++++++
.../cli/datanode/TestDiskBalancerSubCommand.java | 259 +++++++++++++++++++++
9 files changed, 816 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 79cb469047..8f2261a8d3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1309,14 +1309,14 @@ public class SCMClientProtocolServer implements
Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@Override
public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@@ -1326,7 +1326,7 @@ public class SCMClientProtocolServer implements
Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
/**
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
index 8cb2114f57..4c7f71037f 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
@@ -41,7 +41,8 @@ import java.util.concurrent.Callable;
DecommissionSubCommand.class,
MaintenanceSubCommand.class,
RecommissionSubCommand.class,
- UsageInfoSubcommand.class
+ UsageInfoSubcommand.class,
+ DiskBalancerCommands.class
})
@MetaInfServices(SubcommandWithParent.class)
public class DatanodeCommands implements Callable<Void>, SubcommandWithParent {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
new file mode 100644
index 0000000000..649ee59eaa
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand to group disk balancer related operations.
+ *
+ * <p>The balancer is a tool that balances space usage on an Ozone datanode
+ * when some disks become full or when new empty disks were added to a datanode.
+ *
+ * <p>SYNOPSIS
+ * <pre>
+ * To start:
+ * ozone admin datanode diskbalancer start
+ * [ -t/--threshold {@literal <threshold>}]
+ * [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
+ * [ -p/--parallelThread {@literal <parallelThread>}]
+ * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ {@literal <hosts>}]
+ * Examples:
+ * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * start balancer with default values in the configuration on specified
+ * datanodes
+ * ozone admin datanode diskbalancer start -a
+ * start balancer with default values in the configuration on all
+ * datanodes in the cluster
+ * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * start balancer with a threshold of 5%
+ * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * start balancer with maximum 20MB/s diskbandwidth
+ * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * start balancer with 5 parallel thread on each datanode
+ * To stop:
+ * ozone admin datanode diskbalancer stop -a
+ * stop diskblancer on all datanodes
+ * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * stop diskblancer on all datanodes
+ * To update:
+ * ozone admin datanode diskbalancer update -a
+ * update diskblancer configuration on all datanodes
+ * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * update diskblancer configuration on all datanodes
+ * To get report:
+ * ozone admin datanode diskbalancer report -c 10
+ * retrieve at most 10 datanodes that needs diskbalance most
+ * To get status:
+ * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * return the diskbalancer status on datanodes where diskbalancer are in
+ * Running state
+ *
+ * </pre>
+ */
+
+@Command(
+ name = "diskbalancer",
+ description = "DiskBalancer specific operations",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ DiskBalancerStartSubcommand.class,
+ DiskBalancerStopSubcommand.class,
+ DiskBalancerUpdateSubcommand.class,
+ DiskBalancerReportSubcommand.class,
+ DiskBalancerStatusSubcommand.class
+ })
+public class DiskBalancerCommands implements Callable<Void> {
+
+ @Spec
+ private CommandSpec spec;
+
+ @Override
+ public Void call() throws Exception {
+ GenericCli.missingSubcommand(spec);
+ return null;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
new file mode 100644
index 0000000000..deb7b07043
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Handler to get Datanode Volume Density report.
+ */
+@Command(
+ name = "report",
+ description = "Get Datanode Volume Density Report",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerReportSubcommand extends ScmSubcommand {
+ @Option(names = {"-c", "--count"},
+ description = "Result count to return. Sort by Volume Density " +
+ "in descending order.")
+ private int count;
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerReport(count);
+ System.out.println(generateReport(resultProto));
+ }
+
+ private String generateReport(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Report result:%n" +
+ "%-50s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
new file mode 100644
index 0000000000..e3862ba23f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to start disk balancer.
+ */
+@Command(
+ name = "start",
+ description = "Start DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStartSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify " +
+ "\"--allDatanodes\" to start diskBalancer on all datanodes");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Start DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not start DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
new file mode 100644
index 0000000000..6a61876479
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Handler to get disk balancer status.
+ */
+@Command(
+ name = "status",
+ description = "Get Datanode DiskBalancer Status",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStatusSubcommand extends ScmSubcommand {
+
+ private final Set<String> stateSet =
+ new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
+
+ @Option(names = {"-s", "--state"},
+ description = "RUNNING, STOPPED, UNKNOWN")
+ private String state = "RUNNING";
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (state != null && !stateSet.contains(state.toUpperCase(Locale.ROOT))) {
+ System.err.println("Unsupported state: " + state);
+ }
+
+ assert state != null;
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerStatus(
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts),
+ Optional.of(HddsProtos.DiskBalancerRunningStatus.valueOf(
+ state.toUpperCase(Locale.ROOT)))
+ );
+
+ System.out.println(generateStatus(resultProto));
+ }
+
+ private String generateStatus(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Status result:%n" +
+ "%-50s %s %s %s %s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+ contentList.add("Status");
+ contentList.add("Threshold");
+ contentList.add("BandwidthInMB");
+ contentList.add("ParallelThread");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s %s %s %s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ contentList.add(proto.getRunningStatus().name());
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getThreshold()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getDiskBandwidthInMB()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getParallelThread()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
new file mode 100644
index 0000000000..9479c030a6
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to stop disk balancer.
+ */
+@Command(
+ name = "stop",
+ description = "Stop DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStopSubcommand extends ScmSubcommand {
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Stop diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
+ Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Stopping DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not stop DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
new file mode 100644
index 0000000000..b19fb6a53a
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to update disk balancer configuration.
+ */
+@Command(
+ name = "update",
+ description = "Update DiskBalancer Configuration",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
+ parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ throw new IOException(
+ "Some nodes could not update DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
new file mode 100644
index 0000000000..702e6ee7c5
--- /dev/null
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests to validate the DiskBalancerSubCommand class includes the
+ * correct output when executed against a mock client.
+ */
+public class TestDiskBalancerSubCommand {
+
+ private DiskBalancerStopSubcommand stopCmd;
+ private DiskBalancerStartSubcommand startCmd;
+ private DiskBalancerUpdateSubcommand updateCmd;
+ private DiskBalancerReportSubcommand reportCmd;
+ private DiskBalancerStatusSubcommand statusCmd;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ private Random random = new Random();
+
+ @BeforeEach
+ public void setup() throws UnsupportedEncodingException {
+ stopCmd = new DiskBalancerStopSubcommand();
+ startCmd = new DiskBalancerStartSubcommand();
+ updateCmd = new DiskBalancerUpdateSubcommand();
+ reportCmd = new DiskBalancerReportSubcommand();
+ statusCmd = new DiskBalancerStatusSubcommand();
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testDiskBalancerReportSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test report
+ Mockito.when(scmClient.getDiskBalancerReport(Mockito.any(Integer.class)))
+ .thenReturn(generateReport(10));
+
+ reportCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStatusSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test status
+ Mockito.when(scmClient.getDiskBalancerStatus(Mockito.any(), Mockito.any()))
+ .thenReturn(generateStatus(10));
+
+ statusCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStartSubcommand() throws IOException {
+ startCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not start DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ startCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerUpdateSubcommand() throws IOException {
+ updateCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not update DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ updateCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerStopSubcommand() throws IOException {
+ stopCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not stop DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ stopCmd.setAllHosts(false);
+ }
+
+
+ private List<DatanodeAdminError> generateError(int count) {
+ List<DatanodeAdminError> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(new DatanodeAdminError(UUID.randomUUID().toString(),
+ "ERROR"));
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateReport(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateReport());
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateStatus(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateStatus());
+ }
+ return result;
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateReport() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateStatus() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .setRunningStatus(HddsProtos.DiskBalancerRunningStatus.
+ valueOf(random.nextInt(2) + 1))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDetailsProto generateDatanodeDetails() {
+ return HddsProtos.DatanodeDetailsProto.newBuilder()
+ .setHostName(UUID.randomUUID().toString())
+ .setIpAddress("1.1.1.1")
+ .build();
+ }
+
+ private int newLineCount(String str) {
+ int res = 0;
+ String[] lines = str.split("\n");
+ for (String line : lines) {
+ if (line.length() != 0) {
+ res++;
+ }
+ }
+ return res;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 07/07: HDDS-8844. Internal move logic for DiskBalancer (#4887)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 8cc0f2de51c57f4c16a36b791584a384a4a85c7b
Author: Symious <14...@users.noreply.github.com>
AuthorDate: Thu Feb 1 19:53:58 2024 +0800
HDDS-8844. Internal move logic for DiskBalancer (#4887)
---
.../ozone/container/common/impl/ContainerSet.java | 23 ++++
.../container/common/interfaces/Container.java | 11 ++
.../ozone/container/common/interfaces/Handler.java | 14 +++
.../container/common/utils/HddsVolumeUtil.java | 26 ++++
.../diskbalancer/DiskBalancerService.java | 111 +++++++++++++++-
.../policy/ContainerChoosingPolicy.java | 5 +-
.../policy/DefaultContainerChoosingPolicy.java | 17 +--
.../container/keyvalue/KeyValueContainer.java | 140 +++++++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 33 +++++
.../container/ozoneimpl/ContainerController.java | 17 +++
.../diskbalancer/TestDiskBalancerService.java | 36 +++---
.../src/main/proto/DatanodeClientProtocol.proto | 1 +
.../hdds/scm/node/TestNodeDecommissionManager.java | 11 +-
13 files changed, 403 insertions(+), 42 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index b5dfd07d57..b9c3ff3bb5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -116,6 +116,29 @@ public class ContainerSet implements Iterable<Container<?>> {
}
}
+ /**
+ * Update Container to container map.
+ * @param container container to be added
+ * @return If container is added to containerMap returns true, otherwise
+ * false
+ */
+ public Container updateContainer(Container<?> container) throws
+ StorageContainerException {
+ Preconditions.checkNotNull(container, "container cannot be null");
+
+ long containerId = container.getContainerData().getContainerID();
+ if (!containerMap.containsKey(containerId)) {
+ LOG.error("Container doesn't exists with container Id {}", containerId);
+ throw new StorageContainerException("Container doesn't exist with " +
+ "container Id " + containerId,
+ ContainerProtos.Result.CONTAINER_NOT_FOUND);
+ } else {
+ LOG.debug("Container with container Id {} is updated to containerMap",
+ containerId);
+ return containerMap.put(containerId, container);
+ }
+ }
+
/**
* Returns the Container with specified containerId.
* @param containerId ID of the container to get
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
index 5fe148d6aa..94e69f74f2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Path;
import java.time.Instant;
import java.util.Map;
@@ -201,6 +202,11 @@ public interface Container<CONTAINERDATA extends ContainerData> {
void importContainerData(InputStream stream,
ContainerPacker<CONTAINERDATA> packer) throws IOException;
+ /**
+ * Import the container from a container path.
+ */
+ void importContainerData(Path containerPath) throws IOException;
+
/**
* Export all the data of the container to one output archive with the help
* of the packer.
@@ -260,6 +266,11 @@ public interface Container<CONTAINERDATA extends ContainerData> {
ScanResult scanData(DataTransferThrottler throttler, Canceler canceler)
throws InterruptedException;
+ /**
+ * Copy all the data of the container to the destination path.
+ */
+ void copyContainerData(Path destPath) throws IOException;
+
/** Acquire read lock. */
void readLock();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 2ffb9d30d1..f2a96e805f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.container.common.interfaces;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Path;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -217,4 +218,17 @@ public abstract class Handler {
this.clusterId = clusterID;
}
+ /**
+ * Copy container to the destination path.
+ */
+ public abstract void copyContainer(
+ Container container, Path destination)
+ throws IOException;
+
+ /**
+ * Imports container from a container path.
+ */
+ public abstract Container importContainer(
+ ContainerData containerData, Path containerPath) throws IOException;
}
+
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 9b1da77b66..ed1d94e6db 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import java.util.concurrent.CompletableFuture;
import static org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil.onFailure;
@@ -140,4 +141,29 @@ public final class HddsVolumeUtil {
hddsVolume.setDbVolume(globalDbVolumeMap.getOrDefault(
hddsVolume.getStorageID(), null)));
}
+
+
+ /**
+ * Get the HddsVolume according to the path.
+ * @param volumes volume list to match from
+ * @param pathStr path to match
+ */
+ public static HddsVolume matchHddsVolume(List<HddsVolume> volumes,
+ String pathStr) throws IOException {
+ assert pathStr != null;
+ List<HddsVolume> resList = new ArrayList<>();
+ for (HddsVolume hddsVolume: volumes) {
+ if (pathStr.startsWith(hddsVolume.getVolumeRootDir())) {
+ resList.add(hddsVolume);
+ }
+ }
+ if (resList.size() == 1) {
+ return resList.get(0);
+ } else if (resList.size() > 1) {
+ throw new IOException("Get multi volumes " +
+ resList.stream().map(HddsVolume::getVolumeRootDir).collect(
+ Collectors.joining(",")) + " matching path " + pathStr);
+ }
+ return null;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 10980a6b1c..d7498d0ca6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.diskbalancer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -31,13 +32,18 @@ import org.apache.hadoop.hdds.utils.BackgroundTask;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
import org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.util.Time;
import org.apache.ratis.util.FileUtils;
import org.slf4j.Logger;
@@ -45,8 +51,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -327,7 +335,25 @@ public class DiskBalancerService extends BackgroundService {
return queue;
}
- // TODO: Implementation for choose tasks
+ for (int i = 0; i < availableTaskCount; i++) {
+ Pair<HddsVolume, HddsVolume> pair = volumeChoosingPolicy
+ .chooseVolume(volumeSet, threshold, deltaSizes);
+ if (pair == null) {
+ continue;
+ }
+ HddsVolume sourceVolume = pair.getLeft(), destVolume = pair.getRight();
+ ContainerData toBalanceContainer = containerChoosingPolicy
+ .chooseContainer(ozoneContainer, sourceVolume, inProgressContainers);
+ if (toBalanceContainer != null) {
+ queue.add(new DiskBalancerTask(toBalanceContainer, sourceVolume,
+ destVolume));
+ inProgressContainers.add(toBalanceContainer.getContainerID());
+ deltaSizes.put(sourceVolume, deltaSizes.getOrDefault(sourceVolume, 0L)
+ - toBalanceContainer.getBytesUsed());
+ deltaSizes.put(destVolume, deltaSizes.getOrDefault(destVolume, 0L)
+ + toBalanceContainer.getBytesUsed());
+ }
+ }
if (queue.isEmpty()) {
metrics.incrIdleLoopNoAvailableVolumePairCount();
@@ -367,7 +393,88 @@ public class DiskBalancerService extends BackgroundService {
@Override
public BackgroundTaskResult call() {
- // TODO: Details of handling tasks
+ long containerId = containerData.getContainerID();
+ boolean destVolumeIncreased = false;
+ Path diskBalancerTmpDir = null, diskBalancerDestDir = null;
+ long containerSize = containerData.getBytesUsed();
+ try {
+ diskBalancerTmpDir = destVolume.getTmpDir().toPath()
+ .resolve(DISK_BALANCER_DIR).resolve(String.valueOf(containerId));
+
+ // Copy container to new Volume's tmp Dir
+ ozoneContainer.getController().copyContainer(containerData,
+ diskBalancerTmpDir);
+
+ // Move container directory to final place on new volume
+ String idDir = VersionedDatanodeFeatures.ScmHA.chooseContainerPathID(
+ destVolume, destVolume.getClusterID());
+ diskBalancerDestDir =
+ Paths.get(KeyValueContainerLocationUtil.getBaseContainerLocation(
+ destVolume.getHddsRootDir().toString(), idDir,
+ containerData.getContainerID()));
+ Path destDirParent = diskBalancerDestDir.getParent();
+ if (destDirParent != null) {
+ Files.createDirectories(destDirParent);
+ }
+ Files.move(diskBalancerTmpDir, diskBalancerDestDir,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+
+ // Generate a new Container based on destDir
+ File containerFile = ContainerUtils.getContainerFile(
+ diskBalancerDestDir.toFile());
+ if (!containerFile.exists()) {
+ throw new IOException("ContainerFile for container " + containerId
+ + " doesn't exists.");
+ }
+ ContainerData originalContainerData = ContainerDataYaml
+ .readContainerFile(containerFile);
+ Container newContainer = ozoneContainer.getController()
+ .importContainer(originalContainerData, diskBalancerDestDir);
+ newContainer.getContainerData().getVolume()
+ .incrementUsedSpace(containerSize);
+ destVolumeIncreased = true;
+
+ // Update container for containerID
+ Container oldContainer = ozoneContainer.getContainerSet()
+ .getContainer(containerId);
+ oldContainer.writeLock();
+ try {
+ ozoneContainer.getContainerSet().updateContainer(newContainer);
+ oldContainer.delete();
+ } finally {
+ oldContainer.writeUnlock();
+ }
+ oldContainer.getContainerData().getVolume()
+ .decrementUsedSpace(containerSize);
+ metrics.incrSuccessCount(1);
+ metrics.incrSuccessBytes(containerSize);
+ } catch (IOException e) {
+ if (diskBalancerTmpDir != null) {
+ try {
+ Files.deleteIfExists(diskBalancerTmpDir);
+ } catch (IOException ex) {
+ LOG.warn("Failed to delete tmp directory {}", diskBalancerTmpDir,
+ ex);
+ }
+ }
+ if (diskBalancerDestDir != null) {
+ try {
+ Files.deleteIfExists(diskBalancerDestDir);
+ } catch (IOException ex) {
+ LOG.warn("Failed to delete dest directory {}: {}.",
+ diskBalancerDestDir, ex.getMessage());
+ }
+ }
+ // Only need to check for destVolume, sourceVolume's usedSpace is
+ // updated at last, if it reaches there, there is no exception.
+ if (destVolumeIncreased) {
+ destVolume.decrementUsedSpace(containerSize);
+ }
+ metrics.incrFailureCount();
+ } finally {
+ postCall();
+ }
return BackgroundTaskResult.EmptyTaskResult.newResult();
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
index 61b2eeba28..1c699d5305 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import java.util.List;
import java.util.Set;
/**
@@ -34,6 +33,6 @@ public interface ContainerChoosingPolicy {
*
* @return a Container
*/
- List<ContainerData> chooseContainer(OzoneContainer ozoneContainer,
- HddsVolume volume, Set<Long> inProgressContainerIDs, Long targetSize);
+ ContainerData chooseContainer(OzoneContainer ozoneContainer,
+ HddsVolume volume, Set<Long> inProgressContainerIDs);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
index da1e3610ad..f309ac43fa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
@@ -25,9 +25,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
/**
@@ -38,22 +36,17 @@ public class DefaultContainerChoosingPolicy implements ContainerChoosingPolicy {
DefaultContainerChoosingPolicy.class);
@Override
- public List<ContainerData> chooseContainer(OzoneContainer ozoneContainer,
- HddsVolume hddsVolume, Set<Long> inProgressContainerIDs,
- Long targetSize) {
- List<ContainerData> results = new ArrayList<>();
- long sizeTotal = 0L;
-
+ public ContainerData chooseContainer(OzoneContainer ozoneContainer,
+ HddsVolume hddsVolume, Set<Long> inProgressContainerIDs) {
Iterator<Container<?>> itr = ozoneContainer.getController()
.getContainers(hddsVolume);
- while (itr.hasNext() && sizeTotal < targetSize) {
+ while (itr.hasNext()) {
ContainerData containerData = itr.next().getContainerData();
if (!inProgressContainerIDs.contains(
containerData.getContainerID()) && containerData.isClosed()) {
- results.add(containerData);
- sizeTotal += containerData.getBytesUsed();
+ return containerData;
}
}
- return results;
+ return null;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 8388182667..86fed1be73 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_ALREADY_EXISTS;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_DESCRIPTOR_MISSING;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_OPEN;
@@ -679,6 +680,52 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
KeyValueContainerUtil.parseKVContainerData(containerData, config);
}
+ @Override
+ public void importContainerData(Path containerPath) throws IOException {
+ writeLock();
+ try {
+ if (!getContainerFile().exists()) {
+ String errorMessage = String.format(
+ "Can't load container (cid=%d) data from a specific location"
+ + " as the container descriptor (%s) is missing",
+ getContainerData().getContainerID(),
+ getContainerFile().getAbsolutePath());
+ throw new StorageContainerException(errorMessage,
+ CONTAINER_DESCRIPTOR_MISSING);
+ }
+ KeyValueContainerData originalContainerData =
+ (KeyValueContainerData) ContainerDataYaml
+ .readContainerFile(getContainerFile());
+
+ importContainerData(originalContainerData);
+ } catch (Exception ex) {
+ if (ex instanceof StorageContainerException &&
+ ((StorageContainerException) ex).getResult() ==
+ CONTAINER_DESCRIPTOR_MISSING) {
+ throw ex;
+ }
+ //delete all the temporary data in case of any exception.
+ try {
+ if (containerData.getSchemaVersion() != null &&
+ containerData.getSchemaVersion().equals(OzoneConsts.SCHEMA_V3)) {
+ BlockUtils.removeContainerFromDB(containerData, config);
+ }
+ FileUtils.deleteDirectory(new File(containerData.getMetadataPath()));
+ FileUtils.deleteDirectory(new File(containerData.getChunksPath()));
+ FileUtils.deleteDirectory(
+ new File(getContainerData().getContainerPath()));
+ } catch (Exception deleteex) {
+ LOG.error(
+ "Can not cleanup destination directories after a container load"
+ + " error (cid" +
+ containerData.getContainerID() + ")", deleteex);
+ }
+ throw ex;
+ } finally {
+ writeUnlock();
+ }
+ }
+
@Override
public void exportContainerData(OutputStream destination,
ContainerPacker<KeyValueContainerData> packer) throws IOException {
@@ -983,6 +1030,79 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
return checker.fullCheck(throttler, canceler);
}
+ @Override
+ public void copyContainerData(Path destination) throws IOException {
+ writeLock();
+ try {
+ // Closed/ Quasi closed containers are considered for replication by
+ // replication manager if they are under-replicated.
+ ContainerProtos.ContainerDataProto.State state =
+ getContainerData().getState();
+ if (!(state == ContainerProtos.ContainerDataProto.State.CLOSED ||
+ state == ContainerDataProto.State.QUASI_CLOSED)) {
+ throw new IllegalStateException(
+ "Only (quasi)closed containers can be exported, but " +
+ "ContainerId=" + getContainerData().getContainerID() +
+ " is in state " + state);
+ }
+
+ try {
+ if (!containerData.getSchemaVersion().equals(OzoneConsts.SCHEMA_V3)) {
+ compactDB();
+ // Close DB (and remove from cache) to avoid concurrent modification
+ // while copying it.
+ BlockUtils.removeDB(containerData, config);
+ }
+ } finally {
+ readLock();
+ writeUnlock();
+ }
+
+ if (containerData.getSchemaVersion().equals(OzoneConsts.SCHEMA_V3)) {
+ // Synchronize the dump and copy operation,
+ // so concurrent copy don't get dump files overwritten.
+ // We seldom got concurrent exports for a container,
+ // so it should not influence performance much.
+ synchronized (dumpLock) {
+ BlockUtils.dumpKVContainerDataToFiles(containerData, config);
+ copyContainerToDestination(destination);
+ }
+ } else {
+ copyContainerToDestination(destination);
+ }
+ } catch (Exception e) {
+ LOG.error("Got exception when copying container {} to {}",
+ containerData.getContainerID(), destination, e);
+ } finally {
+ if (lock.isWriteLockedByCurrentThread()) {
+ writeUnlock();
+ } else {
+ readUnlock();
+ }
+ }
+ }
+
+ /**
+ * Set all of the path realted container data fields based on the name
+ * conventions.
+ *
+ */
+ public void populatePathFields(HddsVolume volume, Path containerPath) {
+ containerData.setMetadataPath(
+ KeyValueContainerLocationUtil.getContainerMetaDataPath(
+ containerPath.toString()).toString());
+ containerData.setChunksPath(
+ KeyValueContainerLocationUtil.getChunksLocationPath(
+ containerPath.toString()).toString()
+ );
+ containerData.setVolume(volume);
+ containerData.setDbFile(getContainerDBFile());
+ }
+
+ private enum ContainerCheckLevel {
+ NO_CHECK, FAST_CHECK, FULL_CHECK
+ }
+
/**
* Creates a temporary file.
* @param file
@@ -1010,4 +1130,24 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
packer.pack(this, destination);
}
}
+
+ /**
+ * Copy container directory to destination path.
+ * @param destination destination path
+ * @throws IOException file operation exception
+ */
+ private void copyContainerToDestination(Path destination)
+ throws IOException {
+ try {
+ if (Files.exists(destination)) {
+ FileUtils.deleteDirectory(destination.toFile());
+ }
+ FileUtils.copyDirectory(new File(containerData.getContainerPath()),
+ destination.toFile());
+
+ } catch (IOException e) {
+ LOG.error("Failed when copying container to {}", destination, e);
+ throw e;
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index c9d6672ee8..5956d3da6e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
+import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.utils.ContainerLogger;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -1144,6 +1145,38 @@ public class KeyValueHandler extends Handler {
}
}
+ @Override
+ public void copyContainer(final Container container, Path destinationPath)
+ throws IOException {
+ final KeyValueContainer kvc = (KeyValueContainer) container;
+ kvc.copyContainerData(destinationPath);
+ }
+
+ @Override
+ public Container importContainer(ContainerData originalContainerData,
+ final Path containerPath) throws IOException {
+ Preconditions.checkState(originalContainerData instanceof
+ KeyValueContainerData, "Should be KeyValueContainerData instance");
+
+ KeyValueContainerData containerData = new KeyValueContainerData(
+ (KeyValueContainerData) originalContainerData);
+
+ KeyValueContainer container = new KeyValueContainer(containerData,
+ conf);
+
+ HddsVolume volume = HddsVolumeUtil.matchHddsVolume(
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()),
+ containerPath.toString());
+ if (volume == null ||
+ !containerPath.startsWith(volume.getVolumeRootDir())) {
+ throw new IOException("ContainerPath " + containerPath
+ + " doesn't match volume " + volume);
+ }
+ container.populatePathFields(volume, containerPath);
+ container.importContainerData(containerPath);
+ return container;
+ }
+
@Override
public void deleteContainer(Container container, boolean force)
throws IOException {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
index feb5805387..07c627fc99 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Path;
import java.time.Instant;
import java.util.Iterator;
import java.util.Map;
@@ -167,6 +168,22 @@ public class ContainerController {
.importContainer(containerData, rawContainerStream, packer);
}
+ public void copyContainer(final ContainerData containerData,
+ final Path destinationPath) throws IOException {
+ handlers.get(containerData.getContainerType())
+ .copyContainer(
+ containerSet.getContainer(containerData.getContainerID()),
+ destinationPath);
+ }
+
+ public Container importContainer(
+ final ContainerData containerData,
+ final Path containerPath)
+ throws IOException {
+ return handlers.get(containerData.getContainerType())
+ .importContainer(containerData, containerPath);
+ }
+
public void exportContainer(final ContainerType type,
final long containerId, final OutputStream outputStream,
final TarContainerPacker packer) throws IOException {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
index 83aff1a971..664ac7a3ab 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.container.common.TestBlockDeletingService;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@ -37,10 +36,9 @@ import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
import java.io.File;
import java.io.IOException;
@@ -57,28 +55,21 @@ import static org.mockito.Mockito.when;
/**
* This is a test class for DiskBalancerService.
*/
+@Timeout(30)
public class TestDiskBalancerService {
private File testRoot;
private String scmId;
private String datanodeUuid;
- private OzoneConfiguration conf;
+ private OzoneConfiguration conf = new OzoneConfiguration();
- private final ContainerLayoutVersion layout;
- private final String schemaVersion;
+ private ContainerLayoutVersion layout;
+ private String schemaVersion;
private MutableVolumeSet volumeSet;
- public TestDiskBalancerService(ContainerTestVersionInfo versionInfo) {
- this.layout = versionInfo.getLayout();
- this.schemaVersion = versionInfo.getSchemaVersion();
- conf = new OzoneConfiguration();
- ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
- }
-
-
@BeforeEach
public void init() throws IOException {
testRoot = GenericTestUtils
- .getTestDir(TestBlockDeletingService.class.getSimpleName());
+ .getTestDir(TestDiskBalancerService.class.getSimpleName());
if (testRoot.exists()) {
FileUtils.cleanDirectory(testRoot);
}
@@ -100,9 +91,9 @@ public class TestDiskBalancerService {
FileUtils.deleteDirectory(testRoot);
}
- @Timeout(30)
@ContainerTestVersionInfo.ContainerTest
- public void testUpdateService() throws Exception {
+ public void testUpdateService(ContainerTestVersionInfo versionInfo) throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
// Increase volume's usedBytes
for (StorageVolume volume : volumeSet.getVolumeMap().values()) {
volume.incrementUsedSpace(volume.getCapacity() / 2);
@@ -142,8 +133,9 @@ public class TestDiskBalancerService {
svc.shutdown();
}
- @Test
- public void testPolicyClassInitialization() throws IOException {
+ @ContainerTestVersionInfo.ContainerTest
+ public void testPolicyClassInitialization(ContainerTestVersionInfo versionInfo) throws IOException {
+ setLayoutAndSchemaForTest(versionInfo);
ContainerSet containerSet = new ContainerSet(1000);
ContainerMetrics metrics = ContainerMetrics.create(conf);
KeyValueHandler keyValueHandler =
@@ -190,4 +182,10 @@ public class TestDiskBalancerService {
when(ozoneContainer.getController()).thenReturn(controller);
return ozoneContainer;
}
+
+ private void setLayoutAndSchemaForTest(ContainerTestVersionInfo versionInfo) {
+ this.layout = versionInfo.getLayout();
+ this.schemaVersion = versionInfo.getSchemaVersion();
+ ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
+ }
}
diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
index 718e2a108c..cb07f151e9 100644
--- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
@@ -152,6 +152,7 @@ enum Result {
DELETE_ON_NON_EMPTY_CONTAINER = 44;
EXPORT_CONTAINER_METADATA_FAILED = 45;
IMPORT_CONTAINER_METADATA_FAILED = 46;
+ CONTAINER_DESCRIPTOR_MISSING = 47;
}
/**
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
index 09f0dd59b9..eb4f364df8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeUtils.HostDefinition;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -70,21 +71,19 @@ public class TestNodeDecommissionManager {
@Test
public void testHostStringsParseCorrectly()
throws InvalidHostStringException {
- NodeDecommissionManager.HostDefinition def =
- new NodeDecommissionManager.HostDefinition("foobar");
+ HostDefinition def = new HostDefinition("foobar");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(" foobar ");
+ def = new HostDefinition(" foobar ");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition("foobar:1234");
+ def = new HostDefinition("foobar:1234");
assertEquals("foobar", def.getHostname());
assertEquals(1234, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(
- "foobar.mycompany.com:1234");
+ def = new HostDefinition("foobar.mycompany.com:1234");
assertEquals("foobar.mycompany.com", def.getHostname());
assertEquals(1234, def.getPort());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 06/07: HDDS-8182. Add volume and container choosing policy (#4408)
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 9b43e3ef31eb9de0eda35acc4134b4b0da2a0b81
Author: Symious <14...@users.noreply.github.com>
AuthorDate: Mon Apr 10 15:51:27 2023 +0800
HDDS-8182. Add volume and container choosing policy (#4408)
---
.../scm/storage/DiskBalancerConfiguration.java | 23 ++++++++
.../container/common/volume/MutableVolumeSet.java | 11 ++++
.../diskbalancer/DiskBalancerService.java | 24 ++++++++
.../container/diskbalancer/DiskBalancerYaml.java | 4 +-
.../policy/ContainerChoosingPolicy.java | 39 +++++++++++++
.../policy/DefaultContainerChoosingPolicy.java | 59 +++++++++++++++++++
.../policy/DefaultVolumeChoosingPolicy.java | 66 ++++++++++++++++++++++
.../diskbalancer/policy/VolumeChoosingPolicy.java | 41 ++++++++++++++
.../diskbalancer/policy/package-info.java | 22 ++++++++
.../diskbalancer/TestDiskBalancerService.java | 20 +++++++
10 files changed, 307 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index ed40fa30f6..7c218cb241 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -126,6 +126,29 @@ public final class DiskBalancerConfiguration {
this.diskBalancerTimeout = duration.toMillis();
}
+ @Config(key = "volume.choosing.policy", type = ConfigType.CLASS,
+ defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" +
+ ".DefaultVolumeChoosingPolicy",
+ tags = {ConfigTag.DISKBALANCER},
+ description = "The volume choosing policy of the disk balancer service.")
+ private Class<?> volumeChoosingPolicyClass;
+
+ public Class<?> getVolumeChoosingPolicyClass() {
+ return volumeChoosingPolicyClass;
+ }
+
+ @Config(key = "container.choosing.policy", type = ConfigType.CLASS,
+ defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" +
+ ".DefaultContainerChoosingPolicy",
+ tags = {ConfigTag.DISKBALANCER},
+ description = "The container choosing policy of the disk balancer " +
+ "service.")
+ private Class<?> containerChoosingPolicyClass;
+
+ public Class<?> getContainerChoosingPolicyClass() {
+ return containerChoosingPolicyClass;
+ }
+
public DiskBalancerConfiguration() {
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 3c0b6e618e..b103ca7015 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+import org.apache.hadoop.util.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -516,4 +517,14 @@ public class MutableVolumeSet implements VolumeSet {
this.readUnlock();
}
}
+
+ public double getIdealUsage() {
+ long totalCapacity = 0L, totalUsed = 0L;
+ for (StorageVolume volume: volumeMap.values()) {
+ totalCapacity += volume.getCapacity();
+ totalUsed += volume.getUsedSpace();
+ }
+ Preconditions.checkArgument(totalCapacity != 0);
+ return (double) totalUsed / totalCapacity;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 75e1660029..10980a6b1c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.util.Time;
import org.apache.ratis.util.FileUtils;
@@ -89,6 +91,8 @@ public class DiskBalancerService extends BackgroundService {
private Map<HddsVolume, Long> deltaSizes;
private MutableVolumeSet volumeSet;
+ private VolumeChoosingPolicy volumeChoosingPolicy;
+ private ContainerChoosingPolicy containerChoosingPolicy;
private final File diskBalancerInfoFile;
private DiskBalancerServiceMetrics metrics;
@@ -110,6 +114,18 @@ public class DiskBalancerService extends BackgroundService {
deltaSizes = new ConcurrentHashMap<>();
volumeSet = ozoneContainer.getVolumeSet();
+ try {
+ volumeChoosingPolicy = (VolumeChoosingPolicy)
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getVolumeChoosingPolicyClass().newInstance();
+ containerChoosingPolicy = (ContainerChoosingPolicy)
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getContainerChoosingPolicyClass().newInstance();
+ } catch (Exception e) {
+ LOG.error("Got exception when initializing DiskBalancerService", e);
+ throw new RuntimeException(e);
+ }
+
metrics = DiskBalancerServiceMetrics.create();
loadDiskBalancerInfo();
@@ -387,6 +403,14 @@ public class DiskBalancerService extends BackgroundService {
this.balancedBytesInLastWindow.set(bytes);
}
+ public ContainerChoosingPolicy getContainerChoosingPolicy() {
+ return containerChoosingPolicy;
+ }
+
+ public VolumeChoosingPolicy getVolumeChoosingPolicy() {
+ return volumeChoosingPolicy;
+ }
+
@Override
public void shutdown() {
super.shutdown();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
index d16eb65747..d601bc87d8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.container.diskbalancer;
+import org.apache.hadoop.hdds.server.YamlUtils;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
@@ -66,11 +67,10 @@ public final class DiskBalancerYaml {
DiskBalancerInfo diskBalancerInfo;
try (FileInputStream inputFileStream = new FileInputStream(path)) {
- Yaml yaml = new Yaml();
DiskBalancerInfoYaml diskBalancerInfoYaml;
try {
diskBalancerInfoYaml =
- yaml.loadAs(inputFileStream, DiskBalancerInfoYaml.class);
+ YamlUtils.loadAs(inputFileStream, DiskBalancerInfoYaml.class);
} catch (Exception e) {
throw new IOException("Unable to parse yaml file.", e);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
new file mode 100644
index 0000000000..61b2eeba28
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer.policy;
+
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This interface specifies the policy for choosing containers to balance.
+ */
+public interface ContainerChoosingPolicy {
+ /**
+ * Choose a container for balancing.
+ *
+ * @return a Container
+ */
+ List<ContainerData> chooseContainer(OzoneContainer ozoneContainer,
+ HddsVolume volume, Set<Long> inProgressContainerIDs, Long targetSize);
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
new file mode 100644
index 0000000000..da1e3610ad
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer.policy;
+
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Choose a container from specified volume, make sure it's not being balancing.
+ */
+public class DefaultContainerChoosingPolicy implements ContainerChoosingPolicy {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ DefaultContainerChoosingPolicy.class);
+
+ @Override
+ public List<ContainerData> chooseContainer(OzoneContainer ozoneContainer,
+ HddsVolume hddsVolume, Set<Long> inProgressContainerIDs,
+ Long targetSize) {
+ List<ContainerData> results = new ArrayList<>();
+ long sizeTotal = 0L;
+
+ Iterator<Container<?>> itr = ozoneContainer.getController()
+ .getContainers(hddsVolume);
+ while (itr.hasNext() && sizeTotal < targetSize) {
+ ContainerData containerData = itr.next().getContainerData();
+ if (!inProgressContainerIDs.contains(
+ containerData.getContainerID()) && containerData.isClosed()) {
+ results.add(containerData);
+ sizeTotal += containerData.getBytesUsed();
+ }
+ }
+ return results;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
new file mode 100644
index 0000000000..4e2721c811
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer.policy;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Choose a random volume for balancing.
+ */
+public class DefaultVolumeChoosingPolicy implements VolumeChoosingPolicy {
+
+ public static final Logger LOG = LoggerFactory.getLogger(
+ DefaultVolumeChoosingPolicy.class);
+
+ @Override
+ public Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet,
+ double threshold, Map<HddsVolume, Long> deltaMap) {
+ double idealUsage = volumeSet.getIdealUsage();
+
+ List<HddsVolume> volumes = StorageVolumeUtil
+ .getHddsVolumesList(volumeSet.getVolumesList())
+ .stream()
+ .filter(volume -> Math.abs(
+ (double) (volume.getUsedSpace() + deltaMap.getOrDefault(volume, 0L))
+ / volume.getCapacity() - idealUsage) >= threshold)
+ .sorted((v1, v2) ->
+ Double.compare(
+ (double) (v2.getUsedSpace() + deltaMap.getOrDefault(v2, 0L)) /
+ v2.getCapacity(),
+ (double) (v1.getUsedSpace() + deltaMap.getOrDefault(v1, 0L)) /
+ v1.getCapacity()))
+ .collect(Collectors.toList());
+
+ // Can not generate DiskBalancerTask if we have less than 2 results
+ if (volumes.size() <= 1) {
+ LOG.debug("Can not find appropriate Source volume and Dest Volume.");
+ return null;
+ }
+ return Pair.of(volumes.get(0), volumes.get(volumes.size() - 1));
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java
new file mode 100644
index 0000000000..187003bf2f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer.policy;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+
+import java.util.Map;
+
+/**
+ * This interface specifies the policy for choosing volumes to balance.
+ */
+public interface VolumeChoosingPolicy {
+ /**
+ * Choose a pair of volumes for balancing.
+ *
+ * @param volumeSet - volumes to choose from.
+ * @param threshold - the threshold to choose source and dest volumes.
+ * @param deltaSizes - the sizes changes of inProgress balancing jobs.
+ * @return Source volume and Dest volume.
+ */
+ Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet,
+ double threshold, Map<HddsVolume, Long> deltaSizes);
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java
new file mode 100644
index 0000000000..ab9c5f570b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ This package contains policy classes related to the DiskBalancer service.
+ */
+package org.apache.hadoop.ozone.container.diskbalancer.policy;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
index c03f8cab75..83aff1a971 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
+import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
@@ -37,6 +39,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import java.io.File;
@@ -139,6 +142,23 @@ public class TestDiskBalancerService {
svc.shutdown();
}
+ @Test
+ public void testPolicyClassInitialization() throws IOException {
+ ContainerSet containerSet = new ContainerSet(1000);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ KeyValueHandler keyValueHandler =
+ new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
+ metrics, c -> {
+ });
+ DiskBalancerServiceTestImpl svc =
+ getDiskBalancerService(containerSet, conf, keyValueHandler, null, 1);
+
+ assertTrue(svc.getContainerChoosingPolicy()
+ instanceof DefaultContainerChoosingPolicy);
+ assertTrue(svc.getVolumeChoosingPolicy()
+ instanceof DefaultVolumeChoosingPolicy);
+ }
+
private String generateVolumeLocation(String base, int volumeCount) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < volumeCount; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org