You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by yi...@apache.org on 2023/03/16 02:45:04 UTC
[ozone] branch HDDS-5713 updated (1d71713712 -> 7929c5c992)
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a change to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
discard 1d71713712 HDDS-7383. Basic framework of DiskBalancerService (#3874)
omit 6ae2da8218 HDDS-7234. Add a common option for DiskBalancer commands (#3762)
omit 28cfeab74b HDDS-7205. DiskBalancer CLI (#3739)
omit c4a4cc0990 HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
omit 95d9f87f92 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
add 469c034423 HDDS-7687. Support OM transfer Ratis leadership (#4265)
add 55cc72e377 HDDS-7617. Intermittent timeout in TestECContainerRecovery (#4268)
add 22b6852d73 HDDS-7617. Remove flaky tag from TestECContainerRecovery
add e1d4127322 HDDS-7847. Handle Replication of Unhealthy Replicas in RM (#4227)
add 491791c4d3 HDDS-7594. [FSO] Folders created through S3G are created on file system as "files". (#4186)
add 4748fb9726 HDDS-7866. [Snapshot] Fix ConcurrentModificationException in snapdiff API. (#4233)
add b6ad97fd52 HDDS-7962. Add configuration flag to enable LegacyReplicationManager for RATIS containers (#4270)
add 5d5d370de3 HDDS-7919. EC: ECPipelineProvider.createForRead should filter out dead replicas and sort replicas (#4277)
add e7d57b9f91 HDDS-7800. [Snapshot] Use persistent map and set to generate snapshot diff report (#4190)
add 6f6a0571fc HDDS-7964. Enable ratis pre-vote in Ozone by default. (#4272)
add 097594f06b HDDS-7967. Replace Hadoop annotations with Ozone-specific ones (#4279)
add b3b0538597 HDDS-4223. Optimize the construction method of OzoneVolume and OzoneBucket. (#4262)
add 4811cd51d6 HDDS-7865. [Snapshot] Add test for getFileStatus and readFile from snapshot paths. (#4232)
add 3775c4de2b HDDS-7832. Support FSO buckets for FileSizeCountTask. (#4206)
add fc8f79524f HDDS-7931. EC: ManagedChannelImpl not cleaned up properly (#4269)
add bd900ae9db HDDS-7969. CacheValue should not store value as an Optional. (#4275)
add c23ba9a116 HDDS-7556. Translate EC doc into Chinese (#4263)
add cdbe91dd8f HDDS-7980. Force Close QUASI_CLOSED replicas of CLOSED containers in RM (#4283)
add 88c0a6621a HDDS-7976. FsShell creates unrelated trace spans (#4281)
add 23e94b66c0 HDDS-7379. Use certificate bundles instead of the sole certificate (#4231)
add 4261c1df03 HDDS-6572. EC: ReplicationManager - add move manager for container move (#4274)
add 43caa56e18 HDDS-7178. [Multi-Tenant] Use optimistic read in Ranger background sync (#3725)
add 45362b15f0 HDDS-7461. Require CREATE right on volume instead of WRITE for CREATE_BUCKET operation (#3938)
add 59c87ba995 HDDS-7301. Cleanup OmUtils (#3812)
add b466cd582e HDDS-7596. Fix OM crash due to a corner case for FSO-enabled bucket (#4048)
add d2e3e7580d HDDS-7988. Run S3 tests with HA Proxy (#4284)
add 4be2122922 Revert "HDDS-7588. Intermittent failure in TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4040)"
add 204f3b4b97 HDDS-7710. Support AWS s3 ListObjects API's encodingType request parameter (#4127)
add 501152d313 HDDS-7588. Intermittent failure in TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4290)
add cafb372799 HDDS-7064. S3 get-object response emits tracing spans outside ObjectEndpoint#get (#4288)
add 330cec2797 HDDS-7971. Support SCM transfer Ratis leadership (#4276)
add 47a68f8751 HDDS-7989. UnhealthyReplicationProcessor retries failure without delay (#4285)
add 4d09962a65 HDDS-7906. [Snapshot] Wait for checkpoint creation if snapshot in cache and not committed to DB. (#4249)
add c9bbb03556 HDDS-6743. Specify leader node for OM failover (#3409)
add 06e7b1b616 HDDS-7721. Make OM Ratis roles available in /prom endpoint (#4140)
add bcfebad032 HDDS-7721. Make OM Ratis roles available in /prom endpoint (fix compile error)
add 170222e378 HDDS-7991. Do not return fake parent dir for deleted keys (#4287)
add 6b1c62a88f HDDS-7993. [snapshot] Add testcase to handle snapshot name minimum-length (#4293)
add 2fa1e5ab3b HDDS-7998. Synchronize on containerInfo in ReplicationManager and MoveManager (#4295)
add 5da39f8cb8 HDDS-7839. Upgrade Weld to 3.1.9 (#4299)
add 19d9aa38c0 HDDS-7982. NPE in BlockInputStream due to null pipeline after refresh (#4296)
add 73b339a55d HDDS-7983. Intermittent OutOfMemoryError in TestOzoneRpcClientWithRatis#testUploadWithStreamAndMemoryMappedBuffer. (#4292)
add dabe58a2bc HDDS-7782. OM lease recovery for hsync'ed files. (#4255)
add c592d9c857 HDDS-8009. OM HA metrics should be unregistered if leader is not known (#4300)
add 7da66e21ba HDDS-8012. Clean up after link loop test (#4304)
add ec2f36cbd0 HDDS-7973. Let RatisMisReplicationHandler use the new RatisContainerReplicaCount constructor (#4305)
add 4465c494db HDDS-7890. Refactor ContainerDeletionChoosingPolicy implementations (#4303)
add e3868fabdf HDDS-8020. File checksum helper leaking client (#4307)
add 0a66958755 HDDS-8025. ReplicationManager: Count a container once for missing, under, mis or over replicated (#4313)
add aebc06f118 HDDS-8026. Replace import from shaded Guava (#4314)
add dd952d4aae HDDS-8008. Move pendingOps into ContainerStateManagerImpl to ensure consistent state (#4298)
add ce57071a5f HDDS-7590. Use keyManager and trustManager provided by keyStoreFactory in om grpc services (#4145)
add f88b9d0207 HDDS-8035. Mark TestOzoneManagerHAWithData#testOMHAMetrics as flaky
add d75507d7ac HDDS-8035. Mark TestHSync#testO3fsHSync as flaky
add c69a015930 HDDS-8034. Check container replication health before scheduling move in MoveManager (#4321)
add b858170a64 HDDS-8024. Mark TestHSync#testOfsHSync as flaky
add c755e5d645 HDDS-8037. Improve logging in EC Reconstruction putBlock precondition check (#4320)
add e90e2dd8ea HDDS-7505. Recon: Rename OM DB Sync to DB Sync on UI (#3988)
add 628623d664 HDDS-6241. Follower SCM node repeatedly sending requests to Ratis server. (#4302)
add b831d8e6fb HDDS-8036. Unprotected flush in SCMHADBTransactionBuffer (#4319)
add 25ad1f2a4f HDDS-7183. Expose RocksDB critical properties. (#3875)
add a63be2662e HDDS-8044. Ensure GrpcOutputStream is closed (#4327)
add 0349c8e8a9 HDDS-8006. [snapshot] Add unit test cases for snapshot delete (#4297)
add 1b18bd77b3 Revert "HDDS-8006. [snapshot] Add unit test cases for snapshot delete (#4297)" (#4330)
add 9f77588334 HDDS-7959. Improve log in ECBlockInputStream and ECBlockReconstructedStripeInputStream (#4323)
add 9e348de653 HDDS-7906. Addendum: [Snapshot] Wait for checkpoint creation if snapshot in cache and not committed to DB. (#4306)
add abd9dd01f0 HDDS-534. Remove unsupported jmxget subcommand (#4326)
add f0c4d415c7 HDDS-7926. [hsync] Recon throws ClassCastException. (#4266)
add 3d648466ed HDDS-8032. SCM support reconfigurable dynamically (#4318)
add 575cf41950 HDDS-8006. [snapshot] Add unit test cases for snapshot delete (#4331)
add 84f1523d24 HDDS-7869. Log configuration on component startup. (#4271)
add 488e1935a7 HDDS-7156. Reset pending delete block count (#4324)
add 3dd7dc4128 HDDS-8019. IllegalStateException: call already closed at GrpcXceiverService.onCompleted (#4335)
add 8b9d1a852b HDDS-7934. NPE in RandomKeyGenerator's shutdown hook (#4286)
add 3cc079478b HDDS-6176. Ozone service WebUI is not accessible with 404 error. (#4308)
add 1f8795b4fd HDDS-7905. [Snapshot] Snapdiff should only return modifications done to the bucket. (#4273)
add b852395e2a HDDS-8070. DBCheckpointMetrics is not unregistered during OM stop. (#4338)
add 7bdfbba56f HDDS-7984. [Snapshot] Store Ozone keys ever renamed in renamedKeyTable for efficient lookup in SnapshotDeletingService (#4333)
add a01676a9af HDDS-8029. [hsync] Outputstream in encrypted buckets do not return the correct stream capabilities. (#4316)
add a432438c36 HDDS-7210. Missing open containers show up as "Closing" on the container report. (#4207)
add 88292e8a44 HDDS-8075. ECReconstructionCoordinatorTask.runTask should catch Exception (#4342)
add 3bbb5742c5 HDDS-8024. When readChunk from a datanode fails, retry other datanodes. (#4336)
add 360a23c0a1 HDDS-8057. Handle unchecked exception in KeyValueHandler more gracefully (#4334)
add 1af6e91a82 HDDS-8046. Eliminate embedded newlines from config properties (#4328)
add 886fc3d419 HDDS-8077. Enforce new checkstyle: NewlineAtEndOfFile (#4347)
add 30a6d3b472 HDDS-8078. Github CI: Allow PR title starting with Revert (#4348)
add d0dff88fd7 HDDS-8079. RocksDB included in client libs doubles shaded jar size (#4349)
add 50e57f9af5 HDDS-7137. Add CLI for Getting the failed deleted block txn (#3691)
add 28ba9bd9fe HDDS-6802. Implement JSON output for pipeline list CLI (#4343)
add 888c6dca5b HDDS-8073. Replace Usages of LegacyReplicationManager.MoveResult with MoveManager.MoveResult (#4339)
add 4afa68e460 HDDS-7816. Add DataNode list to the SCM WebUI (#4289)
add 6a61108c31 HDDS-8074. Improve synchronization around command queue updates in Node Manager (#4341)
add c168fa06fd HDDS-7950. [Snapshot] API changes to support pagination token and async nature of snapshot diff API (#4264)
add 183600b063 HDDS-5447. HttpFS support in Ozone (#4356)
add cf0235fb02 HDDS-8030. Cleanup unused/unnecessary code related to CertificateClient (#4317)
add 584d68bd51 Revert "HDDS-5447. HttpFS support in Ozone (#4356)"
add 7515008c3b HDDS-5448 Copy over HttpFS module from Hadoop 3.3.1 to Ozone (#2478)
add 409002dd1d HDDS-5519 Remove unnecessary hadoop dependencies from httpfs module (#2536)
add 121b54bd7c HDDS-5520 Add lifecycle management to HttpFS server (#2552)
add 94d16737b2 Merge branch 'master' into HDDS-5447-httpfs
add e8b15f4223 [HTTPFSGW] HDDS-5695 Review ZK and Curator dependencies, and get rid of them. (#2612)
add fbd6cc038f Revert "[HTTPFSGW] HDDS-5695 Review ZK and Curator dependencies, and get rid of them. (#2612)" (#2671)
add b015447715 HDDS-5615 Add a simple test suite for HTTPFS GW. (#2609)
add 8eabe292f5 HDDS-5826 [HTTPFSGW] Remove or replace Hadoop shaded guava dependencies. (#2735)
add 5d671e7462 HDDS-5722 [HTTPFSGGW] junit.jar and json-simple in jar report #2742
add b0236e1976 HDDS-5829 [HTTPFSGW] Move to org.apache.ozone package from org.apache.hadoop. (#2743)
add 0f3e560e18 HDDS-5827 [HTTPFSGW] Remove non-server side related code from Ozone. (#2750)
add 56f104c71e Merge branch 'master' into HDDS-5447-httpfs
add 90d947fe3f Merge branch 'master' into HDDS-5447-httpfs
add f0a7f81de4 Quick fix for HTTPFS gateway dependencies after merge... to be revisited.
add 3ede0703e9 Revert "Quick fix for HTTPFS gateway dependencies after merge... to be revisited."
add 7ba1de2e66 HDDS-6027 [HTTPFSGW] Fix dependency issues after master merge
add bd76bfc70b HDDS-5828 [HTTPFSGW] Add proper handling for unsupported operations (#2862)
add 675e0b9085 HDDS-5698 [HTTPFSGW] Port HTTPFS node and robot tests to ozone-ha, and ozonesecure(-ha) (#2861)
add 08b028967d Merge branch 'master' into HDDS-5447-httpfs
add c7f230f050 Merge branch 'master' into httpfs-merge
add e93762c7b2 HDDS-7719. [HTTPFSGW] Fix secure integration tests for the HttpFS module (#4150)
add b4264b7110 HDDS-7737. [HTTPFSGW] Clean up dependencies (#4199)
add 7563f349ff HDDS-7966. Replace Hadoop annotations with Ozone-specific ones (#4282)
add 78accff380 HDDS-7979. Replace import from shaded Guava (#4310)
add f747312d8c HDDS-8022. Add okio as dependency to the root pom (#4309)
add 75c4553a87 Change ozone-filesystem dependency scope to runtime in the HttpFS module (#4311)
add 69ae8f5ac5 HDDS-8023. [HTTPFSGW] Remove support for satisfyStoragePolicy operation (#4312)
add 8b2a3bd922 HDDS-5966. [HTTPFSGW] Update module doc, and place it in Ozone project docs (#4250)
add 6185f64aab HDDS-8045. Dependency convergence error for zookeeper (#4329)
add a142d70adb Merge branch 'master' into HDDS-5447-httpfs
add 18e9771a0a Merge branch 'master' into HDDS-5447-httpfs
add 47b75d7993 HDDS-8088. [HTTPFSGW] Conform with new checkstyle rule in master #4354
add 66f394b3b5 Merge remote-tracking branch 'origin/HDDS-5447-httpfs'
add c9becb3196 HDDS-8014. [Snapshot] Implement Native Acl Integration tests for Snapshotting (#4301)
add 2094661e52 HDDS-8038. [Snapshot] Add a smoketest for Snapshot with ACLs (#4322)
add 3de525c60c HDDS-8108. gRPC channel created for replication client not shutdown properly (#4363)
add 14d34fb0af HDDS-8112. ECReconstructionCoordinator is not closed (#4365)
add 7849a1202d HDDS-8086. Bump snakeyaml from 1.33 to 2.0 (#4355)
add 2209c3bdd9 HDDS-8087. Intermittent crash in TestHddsDatanodeService (#4353)
add dae6f30a79 HDDS-8095. Unbuffer not supported in TDE buckets (#4361)
add bfc02e4a41 HDDS-8091. Generate list of config tags from ConfigTag enum (#4359)
add 4d025406f6 HDDS-8110. ReplicationManager: Introduce basic limits on ReplicateContainer commands (#4364)
add 9fa04a8f7b close db connection (#4366)
add f77e61ec87 HDDS-8119. Remove loosely related AutoCloseable from SendContainerOutputStream (#4368)
add b1695e30f5 HDDS-8129. ContainerStateMachine allows two different tasks with the same container id running in parallel. (#4370)
add 354d3dc5a1 HDDS-8028. JNI for RocksDB SST Dump tool (#4315)
add 11f3d72784 HDDS-8118. Fail container delete on non empty chunks dir (#4367)
add 111161d782 HDDS-8142. Check if no entries in Block DB for a container on container delete (#4379)
add 3d036f461b HDDS-8133. Create ozone sh key checksum command (#4375)
add 5f6547fc00 HDDS-8131. Add Configuration for OM Ratis Log Purge Tuning Parameters. (#4371)
add ba9c46e29f HDDS-8150. RpcClientTest and ConfigurationSourceTest not run due to naming convention (#4388)
add 5412984766 HDDS-8151. Support fine grained lifetime for root CA certificate (#4386)
add f52cd8b756 HDDS-8144. TestDefaultCertificateClient#testTimeBeforeExpiryGracePeriod fails as we approach DST. (#4382)
add 62692bccbf HDDS-8091. [addendum] Generate list of config tags from ConfigTag enum - Hadoop 3.1 compatibility fix (#4374)
add f83b008a6f HDDS-8076. Use container cache in Key listing API. (#4346)
add 0ebb555fab HDDS-7740. [Snapshot] Implement SnapshotDeletingService (#4244)
add e3206f4e7f HDDS-7930. input stream does not refresh expired block token. (#4378)
add e8c0535392 HDDS-7930. [addendum] input stream does not refresh expired block token.
add f4013866b1 HDDS-8065. Provide GNU long options (#4394)
add 365f80dad2 HDDS-8163 Use try-with-resources to ensure close rockdb connection in SstFilteringService (#4402)
add 2db7dda93a HDDS-8090. When getBlock from a datanode fails, retry other datanodes. (#4357)
add 2fc011762f HDDS-8153. Integrate ContainerBalancer with MoveManager (#4391)
add bf5b6f53ae HDDS-7867. Clean up replication logs (#4397)
add ce418581a8 HDDS-6449. Failed container delete can leave artifacts on disk (#3741)
new 9094b0e839 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
new 248af79181 HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
new 4398f87d62 HDDS-7205. DiskBalancer CLI (#3739)
new 1485590510 HDDS-7234. Add a common option for DiskBalancer commands (#3762)
new 7929c5c992 HDDS-7383. Basic framework of DiskBalancerService (#3874)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1d71713712)
\
N -- N -- N refs/heads/HDDS-5713 (7929c5c992)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/ci/pr_title_check.bats | 28 +
dev-support/ci/pr_title_check.sh | 7 +
.../hadoop/hdds/scm/client/HddsClientUtils.java | 2 +-
.../hadoop/hdds/scm/client/package-info.java | 2 +-
.../org/apache/hadoop/hdds/scm/package-info.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 80 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 4 +-
.../hadoop/hdds/scm/storage/ByteArrayReader.java | 2 +-
.../hadoop/hdds/scm/storage/ByteBufferReader.java | 2 +-
.../hdds/scm/storage/ByteReaderStrategy.java | 2 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 12 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 6 +-
.../hdds/scm/storage/ExtendedInputStream.java | 17 +-
.../hadoop/hdds/scm/storage/StreamBuffer.java | 2 +-
.../hadoop/hdds/scm/storage/package-info.java | 2 +-
.../ozone/client/io/BlockInputStreamFactory.java | 4 +-
.../client/io/BlockInputStreamFactoryImpl.java | 2 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 90 +-
.../ozone/client/io/ECBlockInputStreamFactory.java | 5 +-
.../client/io/ECBlockInputStreamFactoryImpl.java | 3 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 5 +-
.../io/ECBlockReconstructedStripeInputStream.java | 48 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 23 +-
.../hdds/scm/storage/DummyChunkInputStream.java | 2 +-
.../hdds/scm/storage/TestBlockInputStream.java | 126 +-
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
.../hadoop/hdds/scm/storage/TestBufferPool.java | 2 +-
.../hadoop/hdds/scm/storage/package-info.java | 2 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 2 +-
.../ozone/client/io/TestECBlockInputStream.java | 27 +-
.../client/io/TestECBlockInputStreamProxy.java | 3 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 24 +
.../java/org/apache/hadoop/hdds/StringUtils.java | 13 +-
.../hadoop/hdds/annotation/package-info.java | 2 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 7 +-
.../apache/hadoop/hdds/client/package-info.java | 2 +-
.../hadoop/hdds/conf/HddsPrometheusConfig.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 21 +
.../org/apache/hadoop/hdds/conf/package-info.java | 2 +-
.../org/apache/hadoop/hdds/freon/package-info.java | 2 +-
.../java/org/apache/hadoop/hdds/package-info.java | 2 +-
.../apache/hadoop/hdds/protocol/package-info.java | 2 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 59 +
.../hdds/ratis/ServerNotLeaderException.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 24 +-
.../common/helpers/ContainerNotOpenException.java | 2 +-
.../scm/container/common/helpers/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/ha/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 2 +-
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 2 +-
.../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 2 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 5 +-
.../apache/hadoop/hdds/scm/net/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 39 +-
.../hadoop/hdds/scm/pipeline/package-info.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 24 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 125 +-
.../apache/hadoop/hdds/scm/utils/package-info.java | 2 +-
.../hdds/security/exception/package-info.java | 2 +-
.../x509/certificate/utils/CertificateCodec.java | 221 +-
.../hadoop/hdds/security/x509/crl/CRLStatus.java | 2 +-
.../hadoop/hdds/security/x509/package-info.java | 2 +-
.../org/apache/hadoop/hdds/server/YamlUtils.java | 50 +
.../apache/hadoop/hdds/server/package-info.java | 2 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 19 +
.../apache/hadoop/hdds/tracing/package-info.java | 2 +-
.../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 17 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 9 +
.../org/apache/hadoop/ozone/audit/S3GAction.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 4 +-
.../hadoop/ozone/common/ha/ratis/package-info.java | 2 +-
.../apache/hadoop/ozone/common/package-info.java | 2 +-
.../ozone/common/statemachine/StateMachine.java | 2 +-
.../ozone/common/statemachine/package-info.java | 2 +-
.../hadoop/ozone/common/utils/package-info.java | 2 +-
.../org/apache/hadoop/ozone/conf/package-info.java | 2 +-
.../container/common/helpers/package-info.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseException.java | 2 +-
.../org/apache/hadoop/ozone/lock/package-info.java | 2 +-
.../org/apache/hadoop/util/NativeCRC32Wrapper.java | 2 +-
.../main/resources/hdds-version-info.properties | 2 +-
.../common/src/main/resources/ozone-default.xml | 120 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 27 +
.../hdds/client/TestECReplicationConfig.java | 2 +-
.../conf/TestGeneratedConfigurationOverwrite.java | 2 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 16 +
.../org/apache/hadoop/hdds/conf/package-info.java | 2 +-
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 2 +-
.../scm/container/TestContainerReplicaInfo.java | 2 +-
.../org/apache/hadoop/hdds/scm/package-info.java | 2 +-
.../hadoop/hdds/tracing/TestStringCodec.java | 2 +-
.../apache/hadoop/hdds/tracing/package-info.java | 2 +-
.../hadoop/hdds/upgrade/test/MockComponent.java | 2 +-
.../hadoop/hdds/utils/TestHddsIdFactory.java | 2 +-
.../org/apache/hadoop/hdds/utils/package-info.java | 2 +-
.../apache/hadoop/ozone/lock/TestLockManager.java | 2 +-
.../org/apache/hadoop/ozone/lock/package-info.java | 2 +-
.../upgrade/TestLayoutVersionInstanceFactory.java | 2 +-
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 2 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 49 +-
.../org/apache/hadoop/hdds/conf/PostConstruct.java | 2 +-
.../hadoop/hdds/conf/TestConfigFileAppender.java | 2 +-
.../hadoop/hdds/conf/TestConfigFileGenerator.java | 2 +-
...ourceTest.java => TestConfigurationSource.java} | 4 +-
.../java/org/apache/hadoop/ozone/DNMXBeanImpl.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 33 +-
.../helpers/BlockDeletingServiceMetrics.java | 2 +-
.../container/common/helpers/ContainerMetrics.java | 38 +-
.../container/common/helpers/DatanodeIdYaml.java | 4 +-
.../container/common/helpers/package-info.java | 2 +-
.../container/common/impl/ContainerDataYaml.java | 4 +
.../RandomContainerDeletionChoosingPolicy.java | 54 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 57 +-
.../ozone/container/common/impl/package-info.java | 2 +-
.../container/common/interfaces/Container.java | 7 +
.../ContainerDeletionChoosingPolicyTemplate.java} | 80 +-
.../container/common/interfaces/package-info.java | 2 +-
.../ozone/container/common/package-info.java | 2 +-
.../container/common/report/package-info.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 9 +-
.../common/statemachine/DatanodeQueueMetrics.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 7 +-
.../common/statemachine/EndpointStateMachine.java | 2 +-
.../SetNodeOperationalStateCommandHandler.java | 2 +-
.../statemachine/commandhandler/package-info.java | 2 +-
.../common/statemachine/package-info.java | 2 +-
.../common/states/datanode/package-info.java | 2 +-
.../states/endpoint/VersionEndpointTask.java | 17 +-
.../common/states/endpoint/package-info.java | 2 +-
.../transport/server/GrpcXceiverService.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 37 +-
.../common/transport/server/ratis/LocalStream.java | 2 +-
.../container/common/utils/ReferenceCountedDB.java | 2 +-
.../common/utils/db/DatanodeDBProfile.java | 1 +
.../container/common/utils/db/package-info.java | 2 +-
.../ozone/container/common/utils/package-info.java | 2 +-
.../container/common/volume/AbstractFuture.java | 2 +-
.../ozone/container/common/volume/HddsVolume.java | 74 +
.../container/common/volume/TimeoutFuture.java | 2 +-
.../container/common/volume/VolumeInfoMetrics.java | 2 +-
.../container/common/volume/package-info.java | 2 +-
.../reconstruction/ECContainerOperationClient.java | 2 +-
.../ECReconstructionCommandInfo.java | 7 +-
.../ECReconstructionCoordinator.java | 61 +-
.../ECReconstructionCoordinatorTask.java | 9 +-
.../container/ec/reconstruction/package-info.java | 2 +-
.../container/keyvalue/KeyValueContainer.java | 11 +-
.../container/keyvalue/KeyValueContainerCheck.java | 2 +-
.../container/keyvalue/KeyValueContainerData.java | 8 +
.../ozone/container/keyvalue/KeyValueHandler.java | 189 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 206 +-
.../container/keyvalue/helpers/package-info.java | 2 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 2 +-
.../keyvalue/impl/ChunkManagerFactory.java | 2 +-
.../ozone/container/keyvalue/package-info.java | 2 +-
.../background/BlockDeletingService.java | 13 +-
.../statemachine/background/package-info.java | 2 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 2 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 9 +-
.../ozone/container/ozoneimpl/package-info.java | 2 +-
.../replication/AbstractReplicationTask.java | 39 +
.../container/replication/ContainerImporter.java | 2 +-
.../replication/GrpcContainerUploader.java | 44 +-
.../container/replication/GrpcOutputStream.java | 20 +-
.../replication/GrpcReplicationClient.java | 30 +-
.../replication/GrpcReplicationService.java | 9 +-
.../container/replication/PushReplicator.java | 1 +
.../replication/ReplicationSupervisor.java | 21 +-
.../container/replication/ReplicationTask.java | 28 +-
.../replication/SendContainerRequestHandler.java | 15 +-
.../replication/SimpleContainerDownloader.java | 55 +-
.../ozone/container/replication/package-info.java | 2 +-
.../java/org/apache/hadoop/ozone/package-info.java | 2 +-
.../protocol/StorageContainerNodeProtocol.java | 19 +-
.../commands/ReplicateContainerCommand.java | 15 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 4 +
.../commands/SetNodeOperationalStateCommand.java | 2 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 2 +-
.../hdds/datanode/metadata/package-info.java | 2 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 99 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 20 -
.../ozone/container/common/ContainerTestUtils.java | 36 +
.../container/common/TestBlockDeletingService.java | 184 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
.../common/helpers/TestContainerUtils.java | 74 +
.../common/helpers/TestDatanodeIdYaml.java | 40 +-
.../common/impl/TestContainerPersistence.java | 192 +-
.../ozone/container/common/impl/package-info.java | 2 +-
.../ozone/container/common/package-info.java | 2 +-
.../container/common/report/package-info.java | 2 +-
.../TestCreatePipelineCommandHandler.java | 2 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 5 +
.../container/common/volume/TestHddsVolume.java | 10 +
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 4 +
.../common/volume/TestStorageVolumeChecker.java | 2 +-
.../container/common/volume/package-info.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 32 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 24 +
.../ozone/container/keyvalue/package-info.java | 2 +-
.../replication/GrpcOutputStreamTest.java | 11 +
.../replication/TestGrpcContainerUploader.java | 156 ++
.../replication/TestGrpcReplicationService.java | 72 +
.../replication/TestMeasuredReplicator.java | 2 +-
.../replication/TestReplicationSupervisor.java | 32 +-
.../replication/TestSimpleContainerDownloader.java | 218 +-
.../ozone/container/replication/package-info.java | 2 +-
.../stream/TestDirstreamClientHandler.java | 2 +-
.../container/stream/TestStreamingServer.java | 2 +-
.../ozone/container/testutils/package-info.java | 2 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 2 +-
.../TestReconstructionECContainersCommands.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 +-
hadoop-hdds/docs/content/design/httpfs.md | 31 +
.../docs/content/feature/ErasureCoding.zh.md | 168 ++
.../docs/content/feature/Reconfigurability.md | 48 +-
.../docs/content/feature/Reconfigurability.zh.md | 49 +-
hadoop-hdds/docs/content/interface/HttpFS.md | 119 ++
hadoop-hdds/docs/content/tools/_index.md | 2 +-
hadoop-hdds/docs/content/tools/_index.zh.md | 1 -
.../apache/ozone/erasurecode/CodecRegistry.java | 2 +-
.../org/apache/ozone/erasurecode/package-info.java | 4 +-
.../rawcoder/AbstractNativeRawDecoder.java | 2 +-
.../rawcoder/AbstractNativeRawEncoder.java | 2 +-
.../rawcoder/ByteArrayEncodingState.java | 2 +-
.../rawcoder/ByteBufferEncodingState.java | 2 +-
.../erasurecode/rawcoder/DummyRawDecoder.java | 2 +-
.../erasurecode/rawcoder/DummyRawEncoder.java | 2 +-
.../rawcoder/DummyRawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/EncodingState.java | 2 +-
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 2 +-
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 2 +-
.../rawcoder/NativeRSRawErasureCoderFactory.java | 2 +-
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 2 +-
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 2 +-
.../rawcoder/NativeXORRawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 2 +-
.../rawcoder/RSRawErasureCoderFactory.java | 2 +-
.../rawcoder/RawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 2 +-
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 2 +-
.../rawcoder/XORRawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/package-info.java | 4 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 2 +-
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 2 +-
.../ozone/erasurecode/rawcoder/util/GF256.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 2 +-
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 2 +-
.../erasurecode/rawcoder/util/package-info.java | 4 +-
.../rawcoder/TestCodecRawCoderMapping.java | 2 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 2 +-
.../rawcoder/TestNativeXORRawCoder.java | 2 +-
hadoop-hdds/framework/pom.xml | 19 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 4 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../org/apache/hadoop/hdds/conf/package-info.java | 2 +-
.../org/apache/hadoop/hdds/freon/package-info.java | 2 +-
.../hadoop/hdds/protocol/ReconfigureProtocol.java | 10 +-
.../apache/hadoop/hdds/protocol/package-info.java | 2 +-
.../ReconfigureProtocolClientSideTranslatorPB.java | 21 +-
.../ReconfigureProtocolServerSideTranslatorPB.java | 13 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 2 +-
.../hadoop/hdds/protocolPB/package-info.java | 2 +-
.../DeletedBlocksTransactionInfoWrapper.java | 110 +
.../scm/container/common/helpers/package-info.java | 2 +-
.../hadoop/hdds/scm/metadata/package-info.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 29 +
.../hdds/scm/update/client/package-info.java | 2 +-
.../hdds/scm/update/server/package-info.java | 2 +-
.../hadoop/hdds/security/OzoneSecretKey.java | 2 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 17 +-
.../x509/certificate/authority/CAType.java} | 23 +-
.../certificate/authority/CertificateServer.java | 31 +-
.../certificate/authority/DefaultApprover.java | 8 +-
.../certificate/authority/DefaultCAServer.java | 69 +-
.../x509/certificate/authority/package-info.java | 2 +-
.../authority/profile/DefaultCAProfile.java | 2 +-
.../authority/profile/package-info.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 286 +--
.../certificate/client/DNCertificateClient.java | 33 +-
.../client/DefaultCertificateClient.java | 442 ++--
.../certificate/client/SCMCertificateClient.java | 18 +-
.../x509/certificate/client/package-info.java | 2 +-
.../certificate/utils/CertificateSignRequest.java | 55 +-
.../certificate/utils/SelfSignedCertificate.java | 24 +-
.../x509/certificate/utils/package-info.java | 2 +-
.../hadoop/hdds/security/x509/crl/CRLCodec.java | 13 +-
.../hdds/security/x509/exception/package-info.java | 2 +-
.../hdds/security/x509/keys/SecurityUtil.java | 55 +-
.../hdds/security/x509/keys/package-info.java | 2 +-
.../hadoop/hdds/security/x509/package-info.java | 2 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 19 +-
.../apache/hadoop/hdds/server/http/HttpConfig.java | 2 +-
.../hdds/server/http/RatisDropwizardExports.java | 2 +-
.../hadoop/hdds/server/http/package-info.java | 2 +-
.../apache/hadoop/hdds/server/package-info.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 5 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 6 +-
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 160 +-
.../hadoop/hdds/utils/db/DumpFileLoader.java | 2 +-
.../hdds/utils/db/FixedLengthStringCodec.java | 2 +-
.../hdds/utils/db/FixedLengthStringUtils.java | 2 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 3 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 2 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 5 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 2 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 15 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 11 +
.../hadoop/hdds/utils/db/cache/CacheValue.java | 32 +-
.../apache/hadoop/hdds/utils/db/package-info.java | 2 +-
.../org/apache/hadoop/hdds/utils/package-info.java | 2 +-
.../apache/hadoop/ozone/common/package-info.java | 2 +-
.../ssl/TestPemFileBasedKeyStoresFactory.java | 6 +-
.../security/ssl/TestReloadingX509KeyManager.java | 7 +-
.../ssl/TestReloadingX509TrustManager.java | 7 +-
.../token/TestOzoneBlockTokenIdentifier.java | 2 +-
.../token/TestOzoneBlockTokenSecretManager.java | 36 +-
.../hdds/security/x509/CertificateClientTest.java | 273 ---
.../certificate/authority/TestDefaultCAServer.java | 148 +-
.../x509/certificate/authority/package-info.java | 2 +-
.../client/CertificateClientTestImpl.java | 155 +-
.../client/TestDefaultCertificateClient.java | 66 +-
.../client/TestDnCertificateClientInit.java | 2 +-
.../x509/certificate/utils/TestCRLCodec.java | 12 +-
.../certificate/utils/TestCertificateCodec.java | 212 +-
.../utils/TestCertificateSignRequest.java | 13 +-
.../certificate/utils/TestRootCertificate.java | 6 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 2 +-
.../hdds/security/x509/keys/package-info.java | 2 +-
.../hadoop/hdds/security/x509/package-info.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 2 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hadoop/hdds/server/events/package-info.java | 2 +-
.../hdds/server/http/TestBaseHttpServer.java | 2 +-
.../hdds/server/http/TestProfileServlet.java | 2 +-
.../server/http/TestRatisDropwizardExports.java | 2 +-
.../hdds/server/http/TestRatisNameRewrite.java | 2 +-
.../hadoop/hdds/server/http/package-info.java | 2 +-
.../hdds/utils/TestPrometheusMetricsSinkUtil.java | 2 +-
.../hadoop/hdds/utils/TestUgiMetricsUtil.java | 2 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 2 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 2 +-
.../hdds/utils/db/TestFixedLengthStringUtils.java | 2 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/package-info.java | 2 +-
.../apache/hadoop/hdds/utils/db/package-info.java | 2 +-
.../org/apache/hadoop/hdds/utils/package-info.java | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 28 +-
.../src/main/proto/ReconfigureProtocol.proto | 12 +
.../interface-client/src/main/proto/hdds.proto | 14 +
.../dev-support/findbugsExcludeFile.xml | 0
hadoop-hdds/managed-rocksdb/pom.xml | 42 +
.../db/managed/ManagedBlockBasedTableConfig.java | 0
.../hdds/utils/db/managed/ManagedBloomFilter.java | 0
.../hdds/utils/db/managed/ManagedCheckpoint.java | 0
.../db/managed/ManagedColumnFamilyOptions.java | 0
.../db/managed/ManagedCompactRangeOptions.java | 0
.../hdds/utils/db/managed/ManagedDBOptions.java | 0
.../hdds/utils/db/managed/ManagedEnvOptions.java | 0
.../hdds/utils/db/managed/ManagedFlushOptions.java | 0
.../managed/ManagedIngestExternalFileOptions.java | 0
.../hdds/utils/db/managed/ManagedLRUCache.java | 0
.../hdds/utils/db/managed/ManagedObject.java | 0
.../hdds/utils/db/managed/ManagedOptions.java | 0
.../hdds/utils/db/managed/ManagedReadOptions.java | 0
.../hdds/utils/db/managed/ManagedRocksDB.java | 6 +
.../utils/db/managed/ManagedRocksIterator.java | 0
.../db/managed/ManagedRocksObjectMetrics.java | 0
.../utils/db/managed/ManagedRocksObjectUtils.java | 0
.../hadoop/hdds/utils/db/managed/ManagedSlice.java | 0
.../utils/db/managed/ManagedSstFileReader.java} | 16 +-
.../db/managed/ManagedSstFileReaderIterator.java} | 21 +-
.../utils/db/managed/ManagedSstFileWriter.java | 0
.../hdds/utils/db/managed/ManagedStatistics.java | 0
.../db/managed/ManagedTransactionLogIterator.java | 0
.../hdds/utils/db/managed/ManagedWriteBatch.java | 0
.../hdds/utils/db/managed/ManagedWriteOptions.java | 0
.../hadoop/hdds/utils/db/managed/package-info.java | 0
hadoop-hdds/pom.xml | 13 +
hadoop-hdds/rocks-native/pom.xml | 421 ++++
hadoop-hdds/rocks-native/src/CMakeLists.txt | 78 +
.../apache/hadoop/hdds/utils/NativeConstants.java | 13 +-
.../hadoop/hdds/utils/NativeLibraryLoader.java | 150 ++
.../utils/NativeLibraryNotLoadedException.java | 20 +-
.../utils/db/managed/ManagedSSTDumpIterator.java | 225 +++
.../hdds/utils/db/managed/ManagedSSTDumpTool.java | 103 +
.../hdds/utils/db/managed/PipeInputStream.java | 99 +
.../hdds/utils/db/managed}/package-info.java | 6 +-
.../apache/hadoop/hdds/utils}/package-info.java | 6 +-
.../rocks-native/src/main/native/Pipe.cpp | 29 +-
.../rocks-native/src/main/native/Pipe.h | 56 +-
.../src/main/native/PipeInputStream.cpp | 48 +
.../rocks-native/src/main/native/SSTDumpTool.cpp | 48 +
.../src/main/native/cplusplus_to_java_convert.h | 55 +
.../src/main/patches/rocks-native.patch | 522 +++++
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 5 +-
.../apache/ozone/rocksdiff/RelationshipEdge.java | 2 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 49 +-
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 67 +-
.../org/apache/ozone/rocksdiff/package-info.java | 2 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 3 +-
.../block/DatanodeDeletedBlockTransactions.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 12 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 25 +-
.../hadoop/hdds/scm/command/package-info.java | 2 +-
.../container/AbstractContainerReportHandler.java | 2 +-
.../scm/container/CloseContainerEventHandler.java | 7 +-
.../hdds/scm/container/ContainerManagerImpl.java | 13 +-
.../hdds/scm/container/ContainerReplica.java | 12 +
.../scm/container/ContainerStateManagerImpl.java | 33 +-
.../IncrementalContainerReportHandler.java | 11 +-
.../balancer/ContainerBalancerMetrics.java | 15 +-
.../container/balancer/ContainerBalancerTask.java | 159 +-
.../hdds/scm/container/balancer/MoveManager.java | 526 +++++
.../hdds/scm/container/closer/package-info.java | 2 +-
.../hadoop/hdds/scm/container/package-info.java | 2 +-
.../algorithms/SCMContainerPlacementMetrics.java | 2 +-
.../placement/algorithms/package-info.java | 2 +-
.../container/placement/metrics/ContainerStat.java | 2 +-
.../container/placement/metrics/package-info.java | 2 +-
.../hdds/scm/container/placement/package-info.java | 2 +-
...nfo.java => AllSourcesOverloadedException.java} | 14 +-
.../replication/ContainerCheckRequest.java | 2 +-
.../replication/ContainerHealthResult.java | 9 +
.../container/replication/ContainerReplicaOp.java | 4 +-
.../DatanodeCommandCountUpdatedHandler.java | 2 +-
.../replication/ECMisReplicationHandler.java | 6 +-
.../replication/ECUnderReplicationHandler.java | 45 +-
.../LegacyRatisContainerReplicaCount.java | 11 +
.../replication/LegacyReplicationManager.java | 162 +-
.../replication/MisReplicationHandler.java | 41 +-
.../replication/RatisContainerReplicaCount.java | 112 +-
.../replication/RatisMisReplicationHandler.java | 21 +-
.../replication/RatisOverReplicationHandler.java | 87 +-
.../replication/RatisUnderReplicationHandler.java | 101 +-
.../container/replication/ReplicationManager.java | 283 ++-
.../replication/UnhealthyReplicationProcessor.java | 29 +-
.../health/ClosingContainerHandler.java | 2 +-
.../health/DeletingContainerHandler.java | 2 +-
.../health/ECReplicationCheckHandler.java | 8 +-
.../health/MismatchedReplicasHandler.java | 34 +-
.../health/RatisReplicationCheckHandler.java | 9 +-
.../RatisUnhealthyReplicationCheckHandler.java | 196 ++
.../scm/container/replication/package-info.java | 2 +-
.../hdds/scm/container/report/package-info.java | 2 +-
.../hdds/scm/container/states/ContainerState.java | 2 +-
.../hdds/scm/container/states/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 -
.../hadoop/hdds/scm/events/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 62 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 2 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 87 +-
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 27 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 2 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 2 +-
.../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 2 +-
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 97 +-
.../hdds/scm/node/CommandQueueReportHandler.java | 50 -
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../hdds/scm/node/InvalidNodeStateException.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 29 +-
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 7 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 20 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 135 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 2 +-
.../hadoop/hdds/scm/node/StorageReportResult.java | 2 +-
.../apache/hadoop/hdds/scm/node/package-info.java | 2 +-
.../hadoop/hdds/scm/node/states/package-info.java | 2 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 25 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../pipeline/choose/algorithms/package-info.java | 2 +-
.../hadoop/hdds/scm/pipeline/package-info.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 36 +
.../hadoop/hdds/scm/protocol/package-info.java | 2 +-
.../hdds/scm/safemode/SafeModeRestrictedOps.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 113 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 11 +-
.../hdds/scm/server/SCMStarterInterface.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 83 +-
.../scm/server/StorageContainerManagerStarter.java | 4 +-
.../hadoop/hdds/scm/server/package-info.java | 2 +-
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 2 +-
.../hdds/scm/update/server/package-info.java | 2 +-
.../main/resources/webapps/scm/scm-overview.html | 60 +-
.../src/main/resources/webapps/scm/scm.js | 45 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 2 +-
.../scm/TestStorageContainerManagerHttpServer.java | 5 +-
.../apache/hadoop/hdds/scm/block/package-info.java | 2 +-
.../hadoop/hdds/scm/command/package-info.java | 2 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 41 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 34 +-
.../container/TestCloseContainerEventHandler.java | 1 +
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../hdds/scm/container/TestContainerReplica.java | 53 +
.../scm/container/TestContainerReportHandler.java | 7 +-
.../scm/container/TestContainerStateManager.java | 7 +-
.../TestIncrementalContainerReportHandler.java | 7 +-
.../scm/container/TestUnknownContainerReport.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 9 +
.../balancer/TestContainerBalancerTask.java | 168 +-
.../scm/container/balancer/TestMoveManager.java | 514 +++++
.../hdds/scm/container/closer/package-info.java | 2 +-
.../hadoop/hdds/scm/container/package-info.java | 2 +-
.../algorithms/TestContainerPlacementFactory.java | 2 +-
.../TestSCMContainerPlacementCapacity.java | 2 +-
.../TestSCMContainerPlacementRandom.java | 2 +-
.../container/replication/ReplicationTestUtil.java | 28 +
.../replication/TestECMisReplicationHandler.java | 32 +-
.../replication/TestECUnderReplicationHandler.java | 74 +-
.../replication/TestLegacyReplicationManager.java | 102 +-
.../replication/TestMisReplicationHandler.java | 58 +-
.../replication/TestOverReplicatedProcessor.java | 32 +-
.../TestRatisContainerReplicaCount.java | 199 +-
.../TestRatisMisReplicationHandler.java | 30 +-
.../TestRatisOverReplicationHandler.java | 11 +-
.../TestRatisUnderReplicationHandler.java | 67 +-
.../replication/TestReplicationManager.java | 208 +-
.../replication/TestUnderReplicatedProcessor.java | 38 +-
.../health/TestECReplicationCheckHandler.java | 4 +-
.../health/TestMismatchedReplicasHandler.java | 38 +
.../health/TestRatisReplicationCheckHandler.java | 28 +-
.../TestRatisUnhealthyReplicationCheckHandler.java | 334 ++++
.../scm/container/replication/package-info.java | 2 +-
.../hdds/scm/container/report/package-info.java | 2 +-
.../container/states/TestContainerAttribute.java | 2 +-
.../hdds/scm/container/states/package-info.java | 2 +-
.../hadoop/hdds/scm/node/TestCommandQueue.java | 31 +-
.../scm/node/TestCommandQueueReportHandler.java | 135 --
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 2 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 2 +-
.../hadoop/hdds/scm/node/TestNodeStatus.java | 68 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 48 +-
.../apache/hadoop/hdds/scm/node/package-info.java | 2 +-
.../scm/node/states/TestNode2ContainerMap.java | 2 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 2 +-
.../hadoop/hdds/scm/node/states/package-info.java | 2 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 2 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 76 +-
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 2 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 2 +-
.../hdds/scm/pipeline/pipeline/package-info.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 2 +-
.../scm/server/TestSCMBlockProtocolServer.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 43 -
.../scm/server/TestSCMSecurityProtocolServer.java | 2 +-
.../server/TestStorageContainerManagerStarter.java | 2 +-
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +-
.../ozone/container/common/TestEndPoint.java | 140 +-
.../ozone/container/common/package-info.java | 2 +-
.../ozone/container/placement/package-info.java | 2 +-
.../testutils/ReplicationNodeManagerMock.java | 34 +-
.../ozone/container/testutils/package-info.java | 2 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 2 +-
.../hdds/scm/cli/ContainerOperationClient.java | 13 +
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 5 +
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 2 +-
.../hadoop/hdds/scm/cli/cert/InfoSubcommand.java | 4 +-
.../hadoop/hdds/scm/cli/cert/package-info.java | 2 +-
.../hdds/scm/cli/container/package-info.java | 2 +-
.../hadoop/hdds/scm/cli/datanode/package-info.java | 2 +-
.../apache/hadoop/hdds/scm/cli/package-info.java | 2 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 22 +-
.../hadoop/hdds/scm/cli/pipeline/package-info.java | 2 +-
.../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 2 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 2 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 26 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 332 ++-
.../apache/hadoop/ozone/client/OzoneVolume.java | 234 ++-
.../client/checksum/ECFileChecksumHelper.java | 4 +-
.../checksum/ReplicatedFileChecksumHelper.java | 4 +-
.../hadoop/ozone/client/checksum/package-info.java | 2 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 23 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 47 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 9 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 22 +-
.../apache/hadoop/ozone/client/package-info.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 9 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 174 +-
.../hadoop/ozone/client/TestOzoneClient.java | 2 +-
.../TestReplicatedBlockChecksumComputer.java | 2 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 2 +-
.../hadoop/ozone/client/checksum/package-info.java | 2 +-
.../apache/hadoop/ozone/client/package-info.java | 2 +-
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 2 +-
.../rpc/{RpcClientTest.java => TestRpcClient.java} | 6 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 +-
.../ozone/OzoneIllegalArgumentException.java | 2 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +-
.../hadoop/ozone/client/io/package-info.java | 2 +-
.../apache/hadoop/ozone/conf/OMClientConfig.java | 2 +-
.../org/apache/hadoop/ozone/conf/package-info.java | 2 +-
.../org/apache/hadoop/ozone/ha/package-info.java | 2 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 17 +-
.../hadoop/ozone/om/exceptions/OMException.java | 2 +-
.../ozone/om/exceptions/OMNotLeaderException.java | 45 +-
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 15 +-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 12 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 26 +-
.../apache/hadoop/ozone/om/ha/package-info.java | 2 +-
.../ozone/om/helpers/BucketEncryptionKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/KeyValueUtil.java | 2 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 2 +-
.../hadoop/ozone/om/helpers/OmKeyRenameInfo.java | 91 +
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 39 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../hadoop/ozone/om/helpers/package-info.java | 2 +-
.../apache/hadoop/ozone/om/lock/package-info.java | 2 +-
.../om/multitenant/OzoneTenantRolePrincipal.java | 2 +-
.../org/apache/hadoop/ozone/om/package-info.java | 2 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 22 +-
.../hadoop/ozone/om/protocol/package-info.java | 2 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 26 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 76 +-
.../hadoop/ozone/om/protocolPB/package-info.java | 2 +-
.../java/org/apache/hadoop/ozone/package-info.java | 2 +-
.../hadoop/ozone/security/acl/package-info.java | 2 +-
.../apache/hadoop/ozone/security/package-info.java | 2 +-
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 40 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 85 +
.../apache/hadoop/ozone/snapshot/package-info.java | 2 +-
.../org/apache/hadoop/ozone/util/package-info.java | 2 +-
.../ozone/om/helpers/TestOmMultipartUpload.java | 2 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 2 +-
.../org/apache/hadoop/ozone/csi/CsiServer.java | 6 +-
.../org/apache/hadoop/ozone/csi/package-info.java | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 2 +
hadoop-ozone/dist/pom.xml | 16 +-
.../src/main/compose/ozone-ha/docker-compose.yaml | 12 +-
.../dist/src/main/compose/ozone-ha/docker-config | 4 +
.../dist/src/main/compose/ozone-ha/test.sh | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 8 +
.../dist/src/main/compose/ozone/docker-config | 3 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 8 +-
.../dist/src/main/compose/ozones3-haproxy/test.sh | 4 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 18 +
.../src/main/compose/ozonesecure-ha/docker-config | 19 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 2 +
.../main/compose/ozonesecure/docker-compose.yaml | 16 +-
.../src/main/compose/ozonesecure/docker-config | 20 +-
.../dist/src/main/compose/ozonesecure/test.sh | 3 +
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 432 -> 580 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 278 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 146 -> 146 bytes
hadoop-ozone/dist/src/main/keytabs/httpfs.keytab | Bin 0 -> 458 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 142 -> 142 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 278 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 296 -> 296 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 434 -> 434 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 586 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 450 -> 606 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 154 -> 154 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 6 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 9 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 10 +-
.../dist/src/main/smoketest/basic/links.robot | 16 +-
.../src/main/smoketest/ec/backward-compat.robot | 106 +-
.../dist/src/main/smoketest/ec/basic.robot | 89 +-
.../dist/src/main/smoketest/ec/lib.resource | 18 +-
hadoop-ozone/dist/src/main/smoketest/ec/read.robot | 38 +
.../src/main/smoketest/httpfs/operations.robot | 49 +
.../main/smoketest/httpfs/operations_tests.robot | 164 ++
.../dist/src/main/smoketest/omha/testOMHA.robot | 46 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 6 +-
.../dist/src/main/smoketest/s3/webui.robot | 2 +-
.../main/smoketest/snapshot/snapshot-acls.robot | 73 +
hadoop-ozone/dist/src/shell/ozone/ozone | 8 +-
.../apache/hadoop/ozone/failure/package-info.java | 2 +-
.../ozone/loadgenerators/AgedDirLoadGenerator.java | 2 +-
.../loadgenerators/NestedDirLoadGenerator.java | 2 +-
.../loadgenerators/RandomDirLoadGenerator.java | 2 +-
.../src/test/resources/log4j.properties | 2 +-
hadoop-ozone/httpfsgateway/README.txt | 17 +
.../dev-support/findbugsExcludeFile.xml | 41 +
hadoop-ozone/httpfsgateway/pom.xml | 333 ++++
.../httpfsgateway/src/main/conf/httpfs-env.sh | 59 +
.../src/main/conf/httpfs-log4j.properties | 35 +
.../httpfsgateway/src/main/conf/httpfs-site.xml | 17 +
.../org/apache/ozone/fs/http/HttpFSConstants.java | 199 ++
.../org/apache/ozone/fs/http}/package-info.java | 5 +-
.../http/server/CheckUploadContentTypeFilter.java | 115 ++
.../apache/ozone/fs/http/server/FSOperations.java | 2105 ++++++++++++++++++++
.../fs/http/server/HttpFSAuthenticationFilter.java | 144 ++
.../fs/http/server/HttpFSExceptionProvider.java | 113 ++
.../fs/http/server/HttpFSParametersProvider.java | 724 +++++++
.../ozone/fs/http/server/HttpFSReleaseFilter.java | 28 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 1394 +++++++++++++
.../ozone/fs/http/server/HttpFSServerWebApp.java | 162 ++
.../fs/http/server/HttpFSServerWebServer.java | 188 ++
.../org/apache/ozone/fs/http/server/JsonUtil.java | 512 +++++
.../http/server/metrics/HttpFSServerMetrics.java | 164 ++
.../fs/http/server/metrics}/package-info.java | 10 +-
.../apache/ozone/fs/http/server}/package-info.java | 5 +-
.../apache/ozone/hdfs/web/WebHdfsConstants.java | 43 +-
.../org/apache/ozone/hdfs/web}/package-info.java | 5 +-
.../apache/ozone/lib/lang/RunnableCallable.java | 99 +
.../java/org/apache/ozone/lib/lang/XException.java | 140 ++
.../org/apache/ozone/lib/lang}/package-info.java | 5 +-
.../org/apache/ozone/lib/server/BaseService.java | 185 ++
.../java/org/apache/ozone/lib/server/Server.java | 841 ++++++++
.../apache/ozone/lib/server/ServerException.java | 98 +
.../java/org/apache/ozone/lib/server/Service.java | 84 +
.../apache/ozone/lib/server/ServiceException.java | 28 +-
.../org/apache/ozone/lib/server}/package-info.java | 5 +-
.../apache/ozone/lib/service/FileSystemAccess.java | 36 +-
.../lib/service/FileSystemAccessException.java | 63 +
.../java/org/apache/ozone/lib/service/Groups.java} | 22 +-
.../apache/ozone/lib/service/Instrumentation.java | 44 +-
.../org/apache/ozone/lib/service/Scheduler.java | 29 +-
.../service/hadoop/FileSystemAccessService.java | 465 +++++
.../ozone/lib/service/hadoop}/package-info.java | 5 +-
.../instrumentation/InstrumentationService.java | 446 +++++
.../lib/service/instrumentation}/package-info.java | 5 +-
.../apache/ozone/lib/service}/package-info.java | 5 +-
.../lib/service/scheduler/SchedulerService.java | 150 ++
.../ozone/lib/service/scheduler}/package-info.java | 5 +-
.../ozone/lib/service/security/GroupsService.java | 61 +
.../ozone/lib/service/security}/package-info.java | 5 +-
.../ozone/lib/servlet/FileSystemReleaseFilter.java | 115 ++
.../apache/ozone/lib/servlet/HostnameFilter.java | 115 ++
.../org/apache/ozone/lib/servlet/MDCFilter.java | 108 +
.../org/apache/ozone/lib/servlet/ServerWebApp.java | 245 +++
.../apache/ozone/lib/servlet}/package-info.java | 5 +-
.../main/java/org/apache/ozone/lib/util/Check.java | 221 ++
.../apache/ozone/lib/util/ConfigurationUtils.java | 98 +
.../org/apache/ozone/lib/util}/package-info.java | 5 +-
.../org/apache/ozone/lib/wsrs/BooleanParam.java | 37 +-
.../java/org/apache/ozone/lib/wsrs/ByteParam.java} | 30 +-
.../java/org/apache/ozone/lib/wsrs/EnumParam.java | 39 +-
.../org/apache/ozone/lib/wsrs/EnumSetParam.java | 77 +
.../apache/ozone/lib/wsrs/ExceptionProvider.java | 64 +
.../apache/ozone/lib/wsrs/InputStreamEntity.java | 67 +
.../org/apache/ozone/lib/wsrs/IntegerParam.java} | 30 +-
.../org/apache/ozone/lib/wsrs/JSONMapProvider.java | 82 +
.../org/apache/ozone/lib/wsrs/JSONProvider.java | 81 +
.../java/org/apache/ozone/lib/wsrs/LongParam.java} | 30 +-
.../main/java/org/apache/ozone/lib/wsrs/Param.java | 69 +
.../java/org/apache/ozone/lib/wsrs/Parameters.java | 82 +
.../apache/ozone/lib/wsrs/ParametersProvider.java | 128 ++
.../java/org/apache/ozone/lib/wsrs/ShortParam.java | 38 +-
.../org/apache/ozone/lib/wsrs/StringParam.java | 73 +
.../org/apache/ozone/lib/wsrs}/package-info.java | 5 +-
.../main/libexec/shellprofile.d/hadoop-httpfs.sh | 54 +
.../src/main/resources/default-log4j.properties | 20 +
.../src/main/resources/httpfs-default.xml | 337 ++++
.../src/main/resources/httpfs.properties | 21 +
.../src/main/resources/webapps/static/index.html | 36 +
.../main/resources/webapps/webhdfs/WEB-INF/web.xml | 98 +
hadoop-ozone/httpfsgateway/src/main/sbin/httpfs.sh | 64 +
.../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 98 +
.../httpfsgateway/src/site/configuration.xsl | 49 +
.../httpfsgateway/src/site/resources/css/site.css | 30 +
hadoop-ozone/httpfsgateway/src/site/site.xml | 29 +
.../ozone/insight/datanode/package-info.java | 2 +-
.../hadoop/ozone/insight/om/package-info.java | 2 +-
.../apache/hadoop/ozone/insight/package-info.java | 2 +-
.../hadoop/ozone/insight/scm/package-info.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 2 +-
.../ozone/insight/TestConfigurationSubCommand.java | 2 +-
.../hadoop/ozone/insight/TestLogSubcommand.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 7 +
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 227 ++-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 8 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 55 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 45 +-
.../ozone/contract/ITestOzoneContractCreate.java | 23 +-
.../ozone/contract/ITestOzoneContractDelete.java | 23 +-
.../contract/ITestOzoneContractGetFileStatus.java | 28 +-
.../fs/ozone/contract/ITestOzoneContractMkdir.java | 23 +-
.../fs/ozone/contract/ITestOzoneContractOpen.java | 23 +-
.../ozone/contract/ITestOzoneContractRename.java | 23 +-
.../ozone/contract/ITestOzoneContractRootDir.java | 23 +-
.../ozone/contract/ITestOzoneContractUnbuffer.java | 2 +-
.../fs/ozone/contract/ITestOzoneContractUtils.java | 61 -
.../hadoop/fs/ozone/contract/OzoneContract.java | 39 +-
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 2 +-
.../hadoop/hdds/scm/pipeline/package-info.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 397 ++--
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../ozone/TestContainerBalancerOperations.java | 2 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 2 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 110 -
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 233 ++-
.../hadoop/ozone/TestStorageContainerManager.java | 2 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 2 +
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 +
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 +
.../client/rpc/TestContainerStateMachine.java | 2 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 46 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 109 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +
.../ozone/container/TestContainerReplication.java | 50 +-
.../ozone/container/TestECContainerRecovery.java | 10 +-
.../apache/hadoop/ozone/container/TestHelper.java | 20 +-
.../commandhandler/TestDeleteContainerHandler.java | 322 ++-
.../statemachine/commandhandler/package-info.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../metrics/TestDatanodeQueueMetrics.java | 2 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 4 +-
.../hadoop/ozone/container/package-info.java | 2 +-
.../replication/TestContainerReplication.java | 63 +-
.../server/TestSecureContainerServer.java | 4 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 2 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 1 +
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +-
.../ozone/om/TestContainerReportWithKeys.java | 2 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 2 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 2 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 57 +-
.../ozone/om/TestOmContainerLocationCache.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 273 ++-
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 101 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 49 +
.../ozone/om/TestOzoneManagerHAWithData.java | 129 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 1 +
.../hadoop/ozone/om/TestSecureOzoneManager.java | 20 +-
.../org/apache/hadoop/ozone/om/package-info.java | 2 +-
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 543 +++++
.../hadoop/ozone/om/snapshot}/package-info.java | 5 +-
.../java/org/apache/hadoop/ozone/package-info.java | 2 +-
.../hadoop/ozone/reconfig/TestScmReconfigure.java | 101 +
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 71 +-
.../apache/hadoop/ozone/scm/node/package-info.java | 2 +-
.../hadoop/ozone/scm/pipeline/package-info.java | 2 +-
.../ozone/shell/TestDeletedBlocksTxnShell.java | 270 +++
.../TestResetDeletedBlockRetryCountShell.java | 169 --
.../ozone/shell/TestTransferLeadershipShell.java | 137 ++
.../src/main/proto/OmClientProtocol.proto | 42 +-
.../apache/hadoop/ozone/om/ExpiredOpenKeys.java | 68 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 26 +-
.../ozone/om/codec/OmDBDiffReportEntryCodec.java | 53 +
.../ozone/om/codec/OmKeyRenameInfoCodec.java | 55 +
.../hadoop/ozone/om/helpers/package-info.java | 2 +-
.../org/apache/hadoop/ozone/om/package-info.java | 2 +-
.../apache/hadoop/ozone/om/ratis/package-info.java | 2 +-
.../ozone/om/codec/TestOmPrefixInfoCodec.java | 2 +-
.../apache/hadoop/ozone/om/codec/package-info.java | 2 +-
.../hadoop/ozone/om/helpers/package-info.java | 2 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 24 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 11 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 126 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 3 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 2 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 132 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 135 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 139 +-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 9 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 4 +-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 25 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 26 +-
.../hadoop/ozone/om/SstFilteringService.java | 9 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 12 +-
.../apache/hadoop/ozone/om/codec/package-info.java | 2 +-
.../apache/hadoop/ozone/om/fs/package-info.java | 2 +-
.../org/apache/hadoop/ozone/om/ha/OMHAMetrics.java | 132 ++
.../ozone/om/multitenant/AuthorizerLock.java | 15 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 24 +-
.../org/apache/hadoop/ozone/om/package-info.java | 2 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 9 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 71 +-
.../apache/hadoop/ozone/om/ratis/package-info.java | 2 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../om/request/bucket/OMBucketCreateRequest.java | 2 +-
.../ozone/om/request/bucket/package-info.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 14 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 3 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 10 +-
.../request/file/OMFileCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/file/OMFileRequest.java | 72 +-
.../hadoop/ozone/om/request/file/package-info.java | 2 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 13 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 6 +
.../ozone/om/request/key/OMKeyCreateRequest.java | 10 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 4 +
.../om/request/key/acl/prefix/package-info.java | 2 +-
.../hadoop/ozone/om/request/key/package-info.java | 2 +-
.../hadoop/ozone/om/request/package-info.java | 2 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 8 +-
.../S3MultipartUploadCompleteRequest.java | 5 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 5 +-
.../ozone/om/request/security/package-info.java | 2 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 28 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 2 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 138 ++
.../ozone/om/request/snapshot/package-info.java | 2 +-
.../ozone/om/request/validation/package-info.java | 2 +-
.../ozone/om/request/volume/acl/package-info.java | 2 +-
.../ozone/om/request/volume/package-info.java | 2 +-
.../ozone/om/response/bucket/acl/package-info.java | 2 +-
.../ozone/om/response/bucket/package-info.java | 2 +-
.../ozone/om/response/file/package-info.java | 2 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 21 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 19 +-
.../om/response/key/OMKeysDeleteResponse.java | 2 +-
.../om/response/key/OMKeysRenameResponse.java | 22 +-
.../om/response/key/OMTrashRecoverResponse.java | 15 +-
.../hadoop/ozone/om/response/key/package-info.java | 2 +-
.../hadoop/ozone/om/response/package-info.java | 2 +-
.../S3MultipartUploadCompleteResponse.java | 15 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 +-
.../ozone/om/response/security/package-info.java | 2 +-
.../snapshot/OMSnapshotCreateResponse.java | 28 +-
.../OMSnapshotMoveDeletedKeysResponse.java | 126 ++
.../ozone/om/response/volume/package-info.java | 2 +-
.../ozone/om/service/OMRangerBGSyncService.java | 49 +-
.../ozone/om/service/OpenKeyCleanupService.java | 67 +-
.../ozone/om/service/SnapshotDeletingService.java | 380 ++++
.../{package-info.java => PersistentList.java} | 18 +-
.../{package-info.java => PersistentMap.java} | 10 +-
.../{package-info.java => PersistentSet.java} | 12 +-
.../ozone/om/snapshot/RocksDbPersistentList.java | 141 ++
.../ozone/om/snapshot/RocksDbPersistentMap.java | 72 +
.../ozone/om/snapshot/RocksDbPersistentSet.java | 87 +
.../ozone/om/snapshot/SnapshotDiffManager.java | 493 +++--
.../hadoop/ozone/om/snapshot/package-info.java | 2 +-
.../hadoop/ozone/om/upgrade/package-info.java | 2 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 9 +-
.../protocolPB/OzoneManagerRequestHandler.java | 34 +-
.../hadoop/ozone/protocolPB/package-info.java | 2 +-
.../hadoop/ozone/security/OMCertificateClient.java | 83 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 10 +-
.../hadoop/ozone/security/acl/package-info.java | 2 +-
.../apache/hadoop/ozone/security/package-info.java | 2 +-
.../hadoop/ozone/om/TestAuthorizerLockImpl.java | 37 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 10 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 18 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 2 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 46 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 2 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 6 +-
.../apache/hadoop/ozone/om/ha/TestOMHAMetrics.java | 61 +
.../TestMultiTenantAccessController.java | 2 +-
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 12 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 9 +-
.../ozone/om/request/bucket/package-info.java | 2 +-
.../hadoop/ozone/om/request/file/package-info.java | 2 +-
.../hadoop/ozone/om/request/key/package-info.java | 2 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 2 +-
.../ozone/om/request/security/package-info.java | 2 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 94 +
.../snapshot/TestOMSnapshotDeleteRequest.java | 5 +
.../ozone/om/request/snapshot/package-info.java | 2 +-
.../ozone/om/request/volume/acl/package-info.java | 2 +-
.../ozone/om/request/volume/package-info.java | 2 +-
.../ozone/om/response/TestCleanupTableInfo.java | 2 +-
.../ozone/om/response/bucket/package-info.java | 2 +-
.../ozone/om/response/file/package-info.java | 2 +-
.../om/response/key/TestOMKeyRenameResponse.java | 22 +-
.../key/TestOMKeyRenameResponseWithFSO.java | 9 +
.../hadoop/ozone/om/response/key/package-info.java | 2 +-
.../s3/multipart/TestS3MultipartResponse.java | 2 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 85 +
.../ozone/om/response/snapshot/package-info.java | 2 +-
.../ozone/om/service/TestKeyDeletingService.java | 2 +-
.../om/service/TestOpenKeyCleanupService.java | 60 +-
.../om/service/TestSnapshotDeletingService.java | 318 +++
.../om/snapshot/TestRocksDbPersistentList.java | 118 ++
.../om/snapshot/TestRocksDbPersistentMap.java | 127 ++
.../om/snapshot/TestRocksDbPersistentSet.java | 143 ++
.../ozone/om/upgrade/TestOMVersionManager.java | 2 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../security/TestOmCertificateClientInit.java | 12 +-
.../TestOzoneDelegationTokenSecretManager.java | 34 +-
.../ozone/security/TestOzoneManagerBlockToken.java | 2 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 2 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 6 +-
.../fs/ozone/CapableOzoneFSOutputStream.java | 13 +-
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 14 +-
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 34 +
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
.../org/apache/hadoop/fs/StreamCapabilities.java | 132 ++
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
.../apache/hadoop/fs/{ozone => }/package-info.java | 9 +-
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
.../org/apache/hadoop/fs/ozone/package-info.java | 2 +-
hadoop-ozone/pom.xml | 11 +
.../hadoop/ozone/recon/codegen/package-info.java | 2 +-
.../hadoop/ozone/recon/schema/package-info.java | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 14 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 -
.../recon/api/handlers/LegacyBucketHandler.java | 2 +-
.../ozone/recon/api/handlers/package-info.java | 2 +-
.../hadoop/ozone/recon/api/package-info.java | 2 +-
.../hadoop/ozone/recon/api/types/package-info.java | 2 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +-
.../hadoop/ozone/recon/codec/package-info.java | 2 +-
.../ozone/recon/fsck/ContainerHealthStatus.java | 2 +-
.../hadoop/ozone/recon/fsck/package-info.java | 2 +-
.../recon/metrics/ReconTaskStatusMetrics.java | 2 +-
.../apache/hadoop/ozone/recon/package-info.java | 2 +-
.../persistence/SqliteDataSourceProvider.java | 2 +-
.../ozone/recon/persistence/package-info.java | 2 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +-
.../hadoop/ozone/recon/recovery/package-info.java | 2 +-
.../hadoop/ozone/recon/scm/package-info.java | 2 +-
.../recon/security}/ReconCertificateClient.java | 51 +-
.../hadoop/ozone/recon/security}/package-info.java | 4 +-
.../impl/StorageContainerServiceProviderImpl.java | 5 +-
.../hadoop/ozone/recon/spi/impl/package-info.java | 2 +-
.../hadoop/ozone/recon/spi/package-info.java | 2 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 114 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 16 +-
.../hadoop/ozone/recon/tasks/package-info.java | 2 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 44 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 2 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 2 +-
.../hadoop/ozone/recon/api/package-info.java | 2 +-
.../recon/fsck/TestContainerHealthStatus.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../apache/hadoop/ozone/recon/package-info.java | 2 +-
.../ozone/recon/persistence/package-info.java | 2 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 2 +-
.../hadoop/ozone/recon/recovery/package-info.java | 2 +-
.../scm/AbstractReconContainerManagerTest.java | 2 +-
...TestReconIncrementalContainerReportHandler.java | 2 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 2 +-
.../recon/scm/TestReconPipelineReportHandler.java | 2 +-
.../ozone/recon/spi/impl/TestReconDBProvider.java | 2 +-
.../TestStorageContainerServiceProviderImpl.java | 2 +-
.../hadoop/ozone/recon/spi/impl/package-info.java | 2 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 2 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 222 ++-
.../ozone/recon/tasks/TestNSSummaryTask.java | 2 +-
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 2 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 2 +-
.../hadoop/ozone/recon/tasks/package-info.java | 2 +-
hadoop-ozone/s3gateway/pom.xml | 2 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 2 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 9 +-
.../org/apache/hadoop/ozone/s3/TracingFilter.java | 29 +-
.../hadoop/ozone/s3/commontypes/CommonPrefix.java | 8 +-
.../{CommonPrefix.java => EncodingTypeObject.java} | 40 +-
.../hadoop/ozone/s3/commontypes/KeyMetadata.java | 6 +-
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 14 +-
.../hadoop/ozone/s3/commontypes/package-info.java | 2 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 32 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 33 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 18 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 3 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 2 +-
.../hadoop/ozone/s3/metrics/package-info.java | 2 +-
.../org/apache/hadoop/ozone/s3/package-info.java | 2 +-
.../hadoop/ozone/s3/signature/package-info.java | 2 +-
.../apache/hadoop/ozone/s3/util/package-info.java | 2 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 11 +-
.../hadoop/ozone/client/ObjectStoreStub.java | 17 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 86 +-
.../apache/hadoop/ozone/client/package-info.java | 2 +-
.../ozone/s3/TestSignedChunksInputStream.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 23 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 121 +-
.../ozone/s3/endpoint/TestBucketResponse.java | 2 +-
.../TestMultiDeleteRequestUnmarshaller.java | 2 +-
.../ozone/s3/endpoint/TestObjectEndpoint.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 96 +-
.../hadoop/ozone/s3/endpoint/package-info.java | 2 +-
.../hadoop/ozone/s3/exception/package-info.java | 2 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 2 +-
.../org/apache/hadoop/ozone/s3/package-info.java | 2 +-
.../s3/signature/TestStringToSignProducer.java | 2 +-
.../hadoop/ozone/s3/util/TestContinueToken.java | 2 +-
.../hadoop/ozone/s3/util/TestRFC1123Util.java | 2 +-
.../hadoop/ozone/admin/nssummary/package-info.java | 2 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
.../ozone/admin/om/TransferOmLeaderSubCommand.java | 73 +
.../reconfig/ReconfigurePropertiesSubcommand.java | 5 +-
.../admin/reconfig/ReconfigureStartSubcommand.java | 5 +-
.../reconfig/ReconfigureStatusSubcommand.java | 7 +-
...ScmAdmin.java => DeletedBlocksTxnCommands.java} | 40 +-
.../admin/scm/FinalizationScmStatusSubcommand.java | 2 +-
.../scm/GetFailedDeletedBlocksTxnSubcommand.java | 91 +
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 57 +-
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 5 +-
.../admin/scm/TransferScmLeaderSubCommand.java | 72 +
.../org/apache/hadoop/ozone/debug/DBScanner.java | 2 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 2 +-
.../hadoop/ozone/debug/container/package-info.java | 2 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 2 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 7 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../org/apache/hadoop/ozone/fsck/package-info.java | 2 +-
.../hadoop/ozone/segmentparser/BaseLogParser.java | 3 +-
.../ozone/shell/keys/ChecksumKeyHandler.java | 100 +
.../hadoop/ozone/shell/keys/KeyCommands.java | 3 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 12 +-
.../hadoop/ozone/shell/token/TokenOption.java | 2 +-
.../hadoop/ozone/shell/volume/package-info.java | 2 +-
.../ozone/debug/TestDBDefinitionFactory.java | 2 +-
.../containergenerator/TestGeneratorDatanode.java | 2 +-
.../apache/hadoop/ozone/freon/package-info.java | 2 +-
.../org/apache/hadoop/ozone/fsck/package-info.java | 2 +-
.../org/apache/hadoop/ozone/om/package-info.java | 2 +-
.../shell/TestOzoneAddressClientCreation.java | 2 +-
.../ozone/shell/keys/TestChecksumKeyHandler.java | 122 ++
.../hadoop/ozone/shell/keys}/package-info.java | 4 +-
pom.xml | 38 +-
1132 files changed, 33275 insertions(+), 6830 deletions(-)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java
rename hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/{ConfigurationSourceTest.java => TestConfigurationSource.java} (98%)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/{impl/TopNOrderedContainerDeletionChoosingPolicy.java => interfaces/ContainerDeletionChoosingPolicyTemplate.java} (52%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java => hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestDatanodeIdYaml.java (51%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcContainerUploader.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java
create mode 100644 hadoop-hdds/docs/content/design/httpfs.md
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.zh.md
create mode 100644 hadoop-hdds/docs/content/interface/HttpFS.md
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/DeletedBlocksTransactionInfoWrapper.java
copy hadoop-hdds/framework/src/{test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java => main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CAType.java} (67%)
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
rename {hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone => hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate}/client/CertificateClientTestImpl.java (80%)
copy hadoop-hdds/{rocksdb-checkpoint-differ => managed-rocksdb}/dev-support/findbugsExcludeFile.xml (100%)
create mode 100644 hadoop-hdds/managed-rocksdb/pom.xml
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java (93%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java (100%)
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java => managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReader.java} (75%)
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java => managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReaderIterator.java} (71%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java (100%)
rename hadoop-hdds/{framework => managed-rocksdb}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java (100%)
create mode 100644 hadoop-hdds/rocks-native/pom.xml
create mode 100644 hadoop-hdds/rocks-native/src/CMakeLists.txt
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/package-info.java => hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java (77%)
create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
copy hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryNotLoadedException.java (65%)
create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java
create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java
create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature => hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed}/package-info.java (80%)
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature => hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils}/package-info.java (81%)
copy hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => hadoop-hdds/rocks-native/src/main/native/Pipe.cpp (70%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java => hadoop-hdds/rocks-native/src/main/native/Pipe.h (52%)
create mode 100644 hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp
create mode 100644 hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp
create mode 100644 hadoop-hdds/rocks-native/src/main/native/cplusplus_to_java_convert.h
create mode 100644 hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/MoveManager.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{package-info.java => AllSourcesOverloadedException.java} (77%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplica.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestMoveManager.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStatus.java
rename hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/{RpcClientTest.java => TestRpcClient.java} (98%)
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
create mode 100755 hadoop-ozone/dist/src/main/keytabs/httpfs.keytab
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/read.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/httpfs/operations.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-acls.robot
create mode 100644 hadoop-ozone/httpfsgateway/README.txt
create mode 100644 hadoop-ozone/httpfsgateway/dev-support/findbugsExcludeFile.xml
create mode 100644 hadoop-ozone/httpfsgateway/pom.xml
create mode 100644 hadoop-ozone/httpfsgateway/src/main/conf/httpfs-env.sh
create mode 100644 hadoop-ozone/httpfsgateway/src/main/conf/httpfs-log4j.properties
create mode 100644 hadoop-ozone/httpfsgateway/src/main/conf/httpfs-site.xml
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/HttpFSConstants.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http}/package-info.java (88%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/CheckUploadContentTypeFilter.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/FSOperations.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSAuthenticationFilter.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSExceptionProvider.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSParametersProvider.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSReleaseFilter.java (60%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServer.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebApp.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/JsonUtil.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics}/package-info.java (73%)
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server}/package-info.java (89%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/hdfs/web/WebHdfsConstants.java (53%)
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/hdfs/web}/package-info.java (91%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/lang/RunnableCallable.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/lang/XException.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/lang}/package-info.java (92%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server/BaseService.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server/Server.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server/ServerException.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server/Service.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server/ServiceException.java (60%)
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/server}/package-info.java (91%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/FileSystemAccess.java (54%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/FileSystemAccessException.java
copy hadoop-ozone/{ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => httpfsgateway/src/main/java/org/apache/ozone/lib/service/Groups.java} (79%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/Instrumentation.java (56%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/Scheduler.java (63%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop/FileSystemAccessService.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop}/package-info.java (90%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/instrumentation/InstrumentationService.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/instrumentation}/package-info.java (89%)
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service}/package-info.java (89%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/scheduler/SchedulerService.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/scheduler}/package-info.java (91%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/security/GroupsService.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/security}/package-info.java (89%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/servlet/FileSystemReleaseFilter.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/servlet/HostnameFilter.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/servlet/MDCFilter.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/servlet/ServerWebApp.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/servlet}/package-info.java (90%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/util/Check.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/util/ConfigurationUtils.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/util}/package-info.java (93%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/BooleanParam.java (58%)
copy hadoop-ozone/{ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/ByteParam.java} (70%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/EnumParam.java (59%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/EnumSetParam.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/ExceptionProvider.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/InputStreamEntity.java
copy hadoop-ozone/{ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/IntegerParam.java} (69%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/JSONMapProvider.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/JSONProvider.java
copy hadoop-ozone/{ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/package-info.java => httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/LongParam.java} (70%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/Param.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/Parameters.java
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/ParametersProvider.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/ShortParam.java (63%)
create mode 100644 hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/StringParam.java
copy {hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis => hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs}/package-info.java (92%)
create mode 100755 hadoop-ozone/httpfsgateway/src/main/libexec/shellprofile.d/hadoop-httpfs.sh
create mode 100644 hadoop-ozone/httpfsgateway/src/main/resources/default-log4j.properties
create mode 100644 hadoop-ozone/httpfsgateway/src/main/resources/httpfs-default.xml
create mode 100644 hadoop-ozone/httpfsgateway/src/main/resources/httpfs.properties
create mode 100644 hadoop-ozone/httpfsgateway/src/main/resources/webapps/static/index.html
create mode 100644 hadoop-ozone/httpfsgateway/src/main/resources/webapps/webhdfs/WEB-INF/web.xml
create mode 100755 hadoop-ozone/httpfsgateway/src/main/sbin/httpfs.sh
create mode 100644 hadoop-ozone/httpfsgateway/src/main/webapp/WEB-INF/web.xml
create mode 100644 hadoop-ozone/httpfsgateway/src/site/configuration.xsl
create mode 100644 hadoop-ozone/httpfsgateway/src/site/resources/css/site.css
create mode 100644 hadoop-ozone/httpfsgateway/src/site/site.xml
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUtils.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotAcl.java
copy {hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys => hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot}/package-info.java (90%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfigure.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestDeletedBlocksTxnShell.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestResetDeletedBlockRetryCountShell.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestTransferLeadershipShell.java
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ExpiredOpenKeys.java
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/{package-info.java => PersistentList.java} (76%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/{package-info.java => PersistentMap.java} (86%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/{package-info.java => PersistentSet.java} (84%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentMap.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentSet.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentList.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentMap.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentSet.java
create mode 100644 hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
copy hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/{ozone => }/package-info.java (82%)
rename {hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/security}/ReconCertificateClient.java (78%)
copy {hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/security}/package-info.java (89%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/{CommonPrefix.java => EncodingTypeObject.java} (51%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/TransferOmLeaderSubCommand.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/{ScmAdmin.java => DeletedBlocksTxnCommands.java} (57%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetFailedDeletedBlocksTxnSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/TransferScmLeaderSubCommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
copy hadoop-ozone/{recon/src/test/java/org/apache/hadoop/ozone/recon => tools/src/test/java/org/apache/hadoop/ozone/shell/keys}/package-info.java (90%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 03/05: HDDS-7205. DiskBalancer CLI (#3739)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4398f87d6280371a6fbc25de87314860c229e636
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 15 10:02:49 2022 +0800
HDDS-7205. DiskBalancer CLI (#3739)
---
.../hdds/scm/server/SCMClientProtocolServer.java | 6 +-
.../hdds/scm/cli/datanode/DatanodeCommands.java | 3 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 99 ++++++++
.../cli/datanode/DiskBalancerReportSubcommand.java | 70 ++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 100 ++++++++
.../cli/datanode/DiskBalancerStatusSubcommand.java | 103 ++++++++
.../cli/datanode/DiskBalancerStopSubcommand.java | 82 +++++++
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 98 ++++++++
.../cli/datanode/TestDiskBalancerSubCommand.java | 259 +++++++++++++++++++++
9 files changed, 816 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index faa24a8021..3fd3d8d0fb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1248,14 +1248,14 @@ public class SCMClientProtocolServer implements
Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@Override
public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@@ -1265,7 +1265,7 @@ public class SCMClientProtocolServer implements
Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
/**
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
index 8cb2114f57..4c7f71037f 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
@@ -41,7 +41,8 @@ import java.util.concurrent.Callable;
DecommissionSubCommand.class,
MaintenanceSubCommand.class,
RecommissionSubCommand.class,
- UsageInfoSubcommand.class
+ UsageInfoSubcommand.class,
+ DiskBalancerCommands.class
})
@MetaInfServices(SubcommandWithParent.class)
public class DatanodeCommands implements Callable<Void>, SubcommandWithParent {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
new file mode 100644
index 0000000000..649ee59eaa
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand to group disk balancer related operations.
+ *
+ * <p>The balancer is a tool that balances space usage on an Ozone datanode
+ * when some disks become full or when new empty disks were added to a datanode.
+ *
+ * <p>SYNOPSIS
+ * <pre>
+ * To start:
+ * ozone admin datanode diskbalancer start
+ * [ -t/--threshold {@literal <threshold>}]
+ * [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
+ * [ -p/--parallelThread {@literal <parallelThread>}]
+ * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ {@literal <hosts>}]
+ * Examples:
+ * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * start balancer with default values in the configuration on specified
+ * datanodes
+ * ozone admin datanode diskbalancer start -a
+ * start balancer with default values in the configuration on all
+ * datanodes in the cluster
+ * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * start balancer with a threshold of 5%
+ * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * start balancer with maximum 20MB/s diskbandwidth
+ * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * start balancer with 5 parallel thread on each datanode
+ * To stop:
+ * ozone admin datanode diskbalancer stop -a
+ * stop diskblancer on all datanodes
+ * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * stop diskblancer on all datanodes
+ * To update:
+ * ozone admin datanode diskbalancer update -a
+ * update diskblancer configuration on all datanodes
+ * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * update diskblancer configuration on all datanodes
+ * To get report:
+ * ozone admin datanode diskbalancer report -c 10
+ * retrieve at most 10 datanodes that needs diskbalance most
+ * To get status:
+ * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * return the diskbalancer status on datanodes where diskbalancer are in
+ * Running state
+ *
+ * </pre>
+ */
+
+@Command(
+ name = "diskbalancer",
+ description = "DiskBalancer specific operations",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ DiskBalancerStartSubcommand.class,
+ DiskBalancerStopSubcommand.class,
+ DiskBalancerUpdateSubcommand.class,
+ DiskBalancerReportSubcommand.class,
+ DiskBalancerStatusSubcommand.class
+ })
+public class DiskBalancerCommands implements Callable<Void> {
+
+ @Spec
+ private CommandSpec spec;
+
+ @Override
+ public Void call() throws Exception {
+ GenericCli.missingSubcommand(spec);
+ return null;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
new file mode 100644
index 0000000000..deb7b07043
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Handler to get Datanode Volume Density report.
+ */
+@Command(
+ name = "report",
+ description = "Get Datanode Volume Density Report",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerReportSubcommand extends ScmSubcommand {
+ @Option(names = {"-c", "--count"},
+ description = "Result count to return. Sort by Volume Density " +
+ "in descending order.")
+ private int count;
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerReport(count);
+ System.out.println(generateReport(resultProto));
+ }
+
+ private String generateReport(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Report result:%n" +
+ "%-50s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
new file mode 100644
index 0000000000..e3862ba23f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to start disk balancer.
+ */
+@Command(
+ name = "start",
+ description = "Start DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStartSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify " +
+ "\"--allDatanodes\" to start diskBalancer on all datanodes");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Start DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not start DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
new file mode 100644
index 0000000000..6a61876479
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Handler to get disk balancer status.
+ */
+@Command(
+ name = "status",
+ description = "Get Datanode DiskBalancer Status",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStatusSubcommand extends ScmSubcommand {
+
+ private final Set<String> stateSet =
+ new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
+
+ @Option(names = {"-s", "--state"},
+ description = "RUNNING, STOPPED, UNKNOWN")
+ private String state = "RUNNING";
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (state != null && !stateSet.contains(state.toUpperCase(Locale.ROOT))) {
+ System.err.println("Unsupported state: " + state);
+ }
+
+ assert state != null;
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerStatus(
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts),
+ Optional.of(HddsProtos.DiskBalancerRunningStatus.valueOf(
+ state.toUpperCase(Locale.ROOT)))
+ );
+
+ System.out.println(generateStatus(resultProto));
+ }
+
+ private String generateStatus(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Status result:%n" +
+ "%-50s %s %s %s %s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+ contentList.add("Status");
+ contentList.add("Threshold");
+ contentList.add("BandwidthInMB");
+ contentList.add("ParallelThread");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s %s %s %s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ contentList.add(proto.getRunningStatus().name());
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getThreshold()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getDiskBandwidthInMB()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getParallelThread()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
new file mode 100644
index 0000000000..9479c030a6
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to stop disk balancer.
+ */
+@Command(
+ name = "stop",
+ description = "Stop DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStopSubcommand extends ScmSubcommand {
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Stop diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
+ Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Stopping DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not stop DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
new file mode 100644
index 0000000000..b19fb6a53a
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to update disk balancer configuration.
+ */
+@Command(
+ name = "update",
+ description = "Update DiskBalancer Configuration",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
+ parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ throw new IOException(
+ "Some nodes could not update DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
new file mode 100644
index 0000000000..702e6ee7c5
--- /dev/null
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests to validate the DiskBalancerSubCommand class includes the
+ * correct output when executed against a mock client.
+ */
+public class TestDiskBalancerSubCommand {
+
+ private DiskBalancerStopSubcommand stopCmd;
+ private DiskBalancerStartSubcommand startCmd;
+ private DiskBalancerUpdateSubcommand updateCmd;
+ private DiskBalancerReportSubcommand reportCmd;
+ private DiskBalancerStatusSubcommand statusCmd;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ private Random random = new Random();
+
+ @BeforeEach
+ public void setup() throws UnsupportedEncodingException {
+ stopCmd = new DiskBalancerStopSubcommand();
+ startCmd = new DiskBalancerStartSubcommand();
+ updateCmd = new DiskBalancerUpdateSubcommand();
+ reportCmd = new DiskBalancerReportSubcommand();
+ statusCmd = new DiskBalancerStatusSubcommand();
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testDiskBalancerReportSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test report
+ Mockito.when(scmClient.getDiskBalancerReport(Mockito.any(Integer.class)))
+ .thenReturn(generateReport(10));
+
+ reportCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStatusSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test status
+ Mockito.when(scmClient.getDiskBalancerStatus(Mockito.any(), Mockito.any()))
+ .thenReturn(generateStatus(10));
+
+ statusCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStartSubcommand() throws IOException {
+ startCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not start DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ startCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerUpdateSubcommand() throws IOException {
+ updateCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not update DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ updateCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerStopSubcommand() throws IOException {
+ stopCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not stop DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ stopCmd.setAllHosts(false);
+ }
+
+
+ private List<DatanodeAdminError> generateError(int count) {
+ List<DatanodeAdminError> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(new DatanodeAdminError(UUID.randomUUID().toString(),
+ "ERROR"));
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateReport(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateReport());
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateStatus(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateStatus());
+ }
+ return result;
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateReport() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateStatus() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .setRunningStatus(HddsProtos.DiskBalancerRunningStatus.
+ valueOf(random.nextInt(2) + 1))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDetailsProto generateDatanodeDetails() {
+ return HddsProtos.DatanodeDetailsProto.newBuilder()
+ .setHostName(UUID.randomUUID().toString())
+ .setIpAddress("1.1.1.1")
+ .build();
+ }
+
+ private int newLineCount(String str) {
+ int res = 0;
+ String[] lines = str.split("\n");
+ for (String line : lines) {
+ if (line.length() != 0) {
+ res++;
+ }
+ }
+ return res;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 05/05: HDDS-7383. Basic framework of DiskBalancerService (#3874)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7929c5c992b88e648f2b878ea9c5ff6e5292f7c7
Author: Symious <14...@users.noreply.github.com>
AuthorDate: Thu Feb 23 17:10:58 2023 +0800
HDDS-7383. Basic framework of DiskBalancerService (#3874)
---
.../scm/storage/DiskBalancerConfiguration.java | 85 ++++-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../hadoop/hdds/fs/MockSpaceUsageCheckFactory.java | 16 +
.../commandhandler/DiskBalancerCommandHandler.java | 39 +-
.../container/diskbalancer/DiskBalancerInfo.java | 130 +++++++
.../diskbalancer/DiskBalancerService.java | 397 +++++++++++++++++++++
.../diskbalancer/DiskBalancerServiceMetrics.java | 138 +++++++
.../diskbalancer/DiskBalancerVersion.java | 79 ++++
.../container/diskbalancer/DiskBalancerYaml.java | 165 +++++++++
.../ozone/container/diskbalancer/package-info.java | 22 ++
.../ozone/container/ozoneimpl/OzoneContainer.java | 26 +-
.../protocol/commands/DiskBalancerCommand.java | 25 +-
.../diskbalancer/DiskBalancerServiceTestImpl.java | 104 ++++++
.../diskbalancer/TestDiskBalancerService.java | 179 ++++++++++
.../diskbalancer/TestDiskBalancerYaml.java | 53 +++
...inerLocationProtocolClientSideTranslatorPB.java | 7 +-
.../src/main/proto/ScmAdminProtocol.proto | 8 +-
.../interface-client/src/main/proto/hdds.proto | 6 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +-
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 141 ++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 32 +-
22 files changed, 1628 insertions(+), 35 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index 43417f2e81..c41a49db1f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -28,6 +28,11 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Duration;
+import java.util.Optional;
+
+import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE;
+
/**
* This class contains configuration values for the DiskBalancer.
*/
@@ -36,6 +41,17 @@ public final class DiskBalancerConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerConfiguration.class);
+ @Config(key = "info.dir", type = ConfigType.STRING,
+ defaultValue = "", tags = {ConfigTag.DISKBALANCER},
+ description = "The path where datanode diskBalancer's conf is to be " +
+ "written to. if this property is not defined, ozone will fall " +
+ "back to use metadata directory instead.")
+ private String infoDir;
+
+ public String getDiskBalancerInfoDir() {
+ return infoDir;
+ }
+
@Config(key = "volume.density.threshold", type = ConfigType.DOUBLE,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "Threshold is a percentage in the range of 0 to 100. A " +
@@ -55,6 +71,72 @@ public final class DiskBalancerConfiguration {
description = "The max parallel balance thread count.")
private int parallelThread = 5;
+ @Config(key = "should.run.default",
+ defaultValue = "false",
+ type = ConfigType.BOOLEAN,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description =
+ "If DiskBalancer fails to get information from diskbalancer.info, " +
+ "it will choose this value to decide if this service should be " +
+ "running."
+ )
+ private boolean diskBalancerShouldRun = false;
+
+ public boolean getDiskBalancerShouldRun() {
+ return diskBalancerShouldRun;
+ }
+
+ public void setDiskBalancerShouldRun(boolean shouldRun) {
+ this.diskBalancerShouldRun = shouldRun;
+ }
+
+ @Config(key = "service.interval",
+ defaultValue = "60s",
+ type = ConfigType.TIME,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description = "Time interval of the Datanode DiskBalancer service. " +
+ "The Datanode will check the service periodically and update " +
+ "the config and running status for DiskBalancer service. " +
+ "Unit could be defined with postfix (ns,ms,s,m,h,d). "
+ )
+ private long diskBalancerInterval = Duration.ofSeconds(60).toMillis();
+
+ public Duration getDiskBalancerInterval() {
+ return Duration.ofMillis(diskBalancerInterval);
+ }
+
+ public void setDiskBalancerInterval(Duration duration) {
+ this.diskBalancerInterval = duration.toMillis();
+ }
+
+ @Config(key = "service.timeout",
+ defaultValue = "300s",
+ type = ConfigType.TIME,
+ tags = { DATANODE, ConfigTag.DISKBALANCER},
+ description = "Timeout for the Datanode DiskBalancer service. "
+ + "Unit could be defined with postfix (ns,ms,s,m,h,d). "
+ )
+ private long diskBalancerTimeout = Duration.ofSeconds(300).toMillis();
+
+ public Duration getDiskBalancerTimeout() {
+ return Duration.ofMillis(diskBalancerTimeout);
+ }
+
+ public void setDiskBalancerTimeout(Duration duration) {
+ this.diskBalancerTimeout = duration.toMillis();
+ }
+
+ public DiskBalancerConfiguration() {
+ }
+
+ public DiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread) {
+ threshold.ifPresent(aDouble -> this.threshold = aDouble);
+ bandwidthInMB.ifPresent(aLong -> this.diskBandwidthInMB = aLong);
+ parallelThread.ifPresent(integer -> this.parallelThread = integer);
+ }
+
/**
* Gets the threshold value for DiskBalancer.
*
@@ -86,7 +168,8 @@ public final class DiskBalancerConfiguration {
*
* @return max disk bandwidth per second
*/
- public double getDiskBandwidthInMB() {
+
+ public long getDiskBandwidthInMB() {
return diskBandwidthInMB;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 3acc47bc09..bccc8be40f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -248,6 +248,9 @@ public final class OzoneConsts {
*/
public static final String OZONE_SCM_DATANODE_ID_FILE_DEFAULT = "datanode.id";
+ public static final String
+ OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT = "diskBalancer.info";
+
// The ServiceListJSONServlet context attribute where OzoneManager
// instance gets stored.
public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
index 3b2e6415c2..eaeb3c1b4d 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
@@ -50,6 +50,22 @@ public final class MockSpaceUsageCheckFactory {
}
}
+ /**
+ * An implementation that never checks space usage but reports basically
+ * 512G free space. Neither does it persist space usage info.
+ */
+ public static class HalfTera implements SpaceUsageCheckFactory {
+ @Override
+ public SpaceUsageCheckParams paramsFor(File dir) {
+ return new SpaceUsageCheckParams(dir,
+ MockSpaceUsageSource.fixed(512L * 1024 * 1024 * 1024,
+ 512L * 1024 * 1024 * 1024),
+ Duration.ZERO,
+ SpaceUsagePersistence.None.INSTANCE
+ );
+ }
+ }
+
private MockSpaceUsageCheckFactory() {
throw new UnsupportedOperationException("no instances");
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
index 705c0af0e1..d379ef2b69 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
@@ -16,15 +16,21 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerInfo;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -56,7 +62,38 @@ public class DiskBalancerCommandHandler implements CommandHandler {
public void handle(SCMCommand command, OzoneContainer ozoneContainer,
StateContext context, SCMConnectionManager connectionManager) {
invocationCount.incrementAndGet();
- // TODO: Do start/stop/update operation
+ final long startTime = Time.monotonicNow();
+ DiskBalancerCommand diskBalancerCommand = (DiskBalancerCommand) command;
+
+ final HddsProtos.DiskBalancerOpType opType =
+ diskBalancerCommand.getOpType();
+ final DiskBalancerConfiguration diskBalancerConf =
+ diskBalancerCommand.getDiskBalancerConfiguration();
+
+ DiskBalancerInfo diskBalancerInfo = ozoneContainer.getDiskBalancerInfo();
+
+ try {
+ switch (opType) {
+ case START:
+ diskBalancerInfo.setShouldRun(true);
+ diskBalancerInfo.updateFromConf(diskBalancerConf);
+ break;
+ case STOP:
+ diskBalancerInfo.setShouldRun(false);
+ break;
+ case UPDATE:
+ diskBalancerInfo.updateFromConf(diskBalancerConf);
+ break;
+ default:
+ throw new IOException("Unexpected type " + opType);
+ }
+ ozoneContainer.getDiskBalancerService().refresh(diskBalancerInfo);
+ } catch (IOException e) {
+ LOG.error("Can't handle command type: {}", opType, e);
+ } finally {
+ long endTime = Time.monotonicNow();
+ totalTime += endTime - startTime;
+ }
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
new file mode 100644
index 0000000000..873c89aad1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+
+import java.util.Objects;
+
+/**
+ * DiskBalancer's information to persist.
+ */
+public class DiskBalancerInfo {
+ private boolean shouldRun;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+ private DiskBalancerVersion version;
+
+ public DiskBalancerInfo(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread) {
+ this(shouldRun, threshold, bandwidthInMB, parallelThread,
+ DiskBalancerVersion.DEFAULT_VERSION);
+ }
+
+ public DiskBalancerInfo(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread, DiskBalancerVersion version) {
+ this.shouldRun = shouldRun;
+ this.threshold = threshold;
+ this.bandwidthInMB = bandwidthInMB;
+ this.parallelThread = parallelThread;
+ this.version = version;
+ }
+
+ public DiskBalancerInfo(boolean shouldRun,
+ DiskBalancerConfiguration diskBalancerConf) {
+ this.shouldRun = shouldRun;
+ this.threshold = diskBalancerConf.getThreshold();
+ this.bandwidthInMB = diskBalancerConf.getDiskBandwidthInMB();
+ this.parallelThread = diskBalancerConf.getParallelThread();
+ this.version = DiskBalancerVersion.DEFAULT_VERSION;
+ }
+
+ public void updateFromConf(DiskBalancerConfiguration diskBalancerConf) {
+ if (threshold != diskBalancerConf.getThreshold()) {
+ setThreshold(diskBalancerConf.getThreshold());
+ }
+ if (bandwidthInMB != diskBalancerConf.getDiskBandwidthInMB()) {
+ setBandwidthInMB(diskBalancerConf.getDiskBandwidthInMB());
+ }
+ if (parallelThread != diskBalancerConf.getParallelThread()) {
+ setParallelThread(diskBalancerConf.getParallelThread());
+ }
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public long getBandwidthInMB() {
+ return bandwidthInMB;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public int getParallelThread() {
+ return parallelThread;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public DiskBalancerVersion getVersion() {
+ return version;
+ };
+
+ public void setVersion(DiskBalancerVersion version) {
+ this.version = version;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DiskBalancerInfo that = (DiskBalancerInfo) o;
+ return shouldRun == that.shouldRun &&
+ Double.compare(that.threshold, threshold) == 0 &&
+ bandwidthInMB == that.bandwidthInMB &&
+ parallelThread == that.parallelThread &&
+ version == that.version;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(shouldRun, threshold, bandwidthInMB, parallelThread,
+ version);
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
new file mode 100644
index 0000000000..75e1660029
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.util.Time;
+import org.apache.ratis.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A per-datanode disk balancing service takes in charge
+ * of moving contains among disks.
+ */
+public class DiskBalancerService extends BackgroundService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerService.class);
+
+ public static final String DISK_BALANCER_DIR = "diskBalancer";
+
+ private static final String DISK_BALANCER_TMP_DIR = "tmp";
+
+ private OzoneContainer ozoneContainer;
+ private final ConfigurationSource conf;
+
+ private boolean shouldRun = false;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+
+ private DiskBalancerVersion version;
+
+ private AtomicLong totalBalancedBytes = new AtomicLong(0L);
+ private AtomicLong balancedBytesInLastWindow = new AtomicLong(0L);
+ private AtomicLong nextAvailableTime = new AtomicLong(Time.monotonicNow());
+
+ private Map<DiskBalancerTask, Integer> inProgressTasks;
+ private Set<Long> inProgressContainers;
+
+ // Every time a container is decided to be moved from Vol A to Vol B,
+ // the size will be deducted from Vol A and added to Vol B.
+ // This map is used to help calculate the expected storage size after
+ // the container balancing finished.
+ private Map<HddsVolume, Long> deltaSizes;
+ private MutableVolumeSet volumeSet;
+
+ private final File diskBalancerInfoFile;
+
+ private DiskBalancerServiceMetrics metrics;
+
+ public DiskBalancerService(OzoneContainer ozoneContainer,
+ long serviceCheckInterval, long serviceCheckTimeout, TimeUnit timeUnit,
+ int workerSize, ConfigurationSource conf) throws IOException {
+ super("DiskBalancerService", serviceCheckInterval, timeUnit, workerSize,
+ serviceCheckTimeout);
+ this.ozoneContainer = ozoneContainer;
+ this.conf = conf;
+
+ String diskBalancerInfoPath = getDiskBalancerInfoPath();
+ Preconditions.checkNotNull(diskBalancerInfoPath);
+ diskBalancerInfoFile = new File(diskBalancerInfoPath);
+
+ inProgressTasks = new ConcurrentHashMap<>();
+ inProgressContainers = ConcurrentHashMap.newKeySet();
+ deltaSizes = new ConcurrentHashMap<>();
+ volumeSet = ozoneContainer.getVolumeSet();
+
+ metrics = DiskBalancerServiceMetrics.create();
+
+ loadDiskBalancerInfo();
+
+ constructTmpDir();
+ }
+
+ /**
+ * Update DiskBalancerService based on new DiskBalancerInfo.
+ * @param diskBalancerInfo
+ * @throws IOException
+ */
+ public void refresh(DiskBalancerInfo diskBalancerInfo) throws IOException {
+ applyDiskBalancerInfo(diskBalancerInfo);
+ }
+
+ private void constructTmpDir() throws IOException {
+ for (HddsVolume volume:
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())) {
+ Path tmpDir = getDiskBalancerTmpDir(volume);
+ try {
+ FileUtils.deleteFully(tmpDir);
+ FileUtils.createDirectories(tmpDir);
+ } catch (IOException ex) {
+ LOG.warn("Can not reconstruct tmp directory under volume {}", volume,
+ ex);
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * If the diskBalancer.info file exists, load the file. If not exists,
+ * return the default config.
+ * @throws IOException
+ */
+ private void loadDiskBalancerInfo() throws IOException {
+ DiskBalancerInfo diskBalancerInfo;
+ try {
+ if (diskBalancerInfoFile.exists()) {
+ diskBalancerInfo = readDiskBalancerInfoFile(diskBalancerInfoFile);
+ } else {
+ boolean shouldRunDefault =
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getDiskBalancerShouldRun();
+ diskBalancerInfo = new DiskBalancerInfo(shouldRunDefault,
+ new DiskBalancerConfiguration());
+ }
+ } catch (IOException e) {
+ LOG.warn("Can not load diskBalancerInfo from diskBalancer.info file. " +
+ "Falling back to default configs", e);
+ throw e;
+ }
+
+ applyDiskBalancerInfo(diskBalancerInfo);
+ }
+
+ private void applyDiskBalancerInfo(DiskBalancerInfo diskBalancerInfo)
+ throws IOException {
+ // First store in local file, then update in memory variables
+ writeDiskBalancerInfoTo(diskBalancerInfo, diskBalancerInfoFile);
+
+ setShouldRun(diskBalancerInfo.isShouldRun());
+ setThreshold(diskBalancerInfo.getThreshold());
+ setBandwidthInMB(diskBalancerInfo.getBandwidthInMB());
+ setParallelThread(diskBalancerInfo.getParallelThread());
+ setVersion(diskBalancerInfo.getVersion());
+
+ // Default executorService is ScheduledThreadPoolExecutor, so we can
+ // update the poll size by setting corePoolSize.
+ if ((getExecutorService() instanceof ScheduledThreadPoolExecutor)) {
+ ((ScheduledThreadPoolExecutor) getExecutorService())
+ .setCorePoolSize(parallelThread);
+ }
+ }
+
+ private String getDiskBalancerInfoPath() {
+ String diskBalancerInfoDir =
+ conf.getObject(DiskBalancerConfiguration.class)
+ .getDiskBalancerInfoDir();
+ if (Strings.isNullOrEmpty(diskBalancerInfoDir)) {
+ File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
+ if (metaDirPath == null) {
+ // this means meta data is not found, in theory should not happen at
+ // this point because should've failed earlier.
+ throw new IllegalArgumentException("Unable to locate meta data" +
+ "directory when getting datanode disk balancer file path");
+ }
+ diskBalancerInfoDir = metaDirPath.toString();
+ }
+ // Use default datanode disk balancer file name for file path
+ return new File(diskBalancerInfoDir,
+ OzoneConsts.OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT)
+ .toString();
+ }
+
+ /**
+ * Read {@link DiskBalancerInfo} from a local info file.
+ *
+ * @param path DiskBalancerInfo file local path
+ * @return {@link DatanodeDetails}
+ * @throws IOException If the conf file is malformed or other I/O exceptions
+ */
+ private synchronized DiskBalancerInfo readDiskBalancerInfoFile(
+ File path) throws IOException {
+ if (!path.exists()) {
+ throw new IOException("DiskBalancerConf file not found.");
+ }
+ try {
+ return DiskBalancerYaml.readDiskBalancerInfoFile(path);
+ } catch (IOException e) {
+ LOG.warn("Error loading DiskBalancerInfo yaml from {}",
+ path.getAbsolutePath(), e);
+ throw new IOException("Failed to parse DiskBalancerInfo from "
+ + path.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Persistent a {@link DiskBalancerInfo} to a local file.
+ *
+ * @throws IOException when read/write error occurs
+ */
+ private synchronized void writeDiskBalancerInfoTo(
+ DiskBalancerInfo diskBalancerInfo, File path)
+ throws IOException {
+ if (path.exists()) {
+ if (!path.delete() || !path.createNewFile()) {
+ throw new IOException("Unable to overwrite the DiskBalancerInfo file.");
+ }
+ } else {
+ if (!path.getParentFile().exists() &&
+ !path.getParentFile().mkdirs()) {
+ throw new IOException("Unable to create DiskBalancerInfo directories.");
+ }
+ }
+ DiskBalancerYaml.createDiskBalancerInfoFile(diskBalancerInfo, path);
+ }
+
+
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public void setVersion(DiskBalancerVersion version) {
+ this.version = version;
+ }
+
+ public DiskBalancerInfo getDiskBalancerInfo() {
+ return new DiskBalancerInfo(shouldRun, threshold, bandwidthInMB,
+ parallelThread, version);
+ }
+
+ public DiskBalancerReportProto getDiskBalancerReportProto() {
+ DiskBalancerReportProto.Builder builder =
+ DiskBalancerReportProto.newBuilder();
+ return builder.setIsRunning(shouldRun)
+ .setBalancedBytes(totalBalancedBytes.get())
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(threshold)
+ .setDiskBandwidthInMB(bandwidthInMB)
+ .setParallelThread(parallelThread)
+ .build())
+ .build();
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue queue = new BackgroundTaskQueue();
+
+ if (!shouldRun) {
+ return queue;
+ }
+ metrics.incrRunningLoopCount();
+
+ if (shouldDelay()) {
+ metrics.incrIdleLoopExceedsBandwidthCount();
+ return queue;
+ }
+
+ int availableTaskCount = parallelThread - inProgressTasks.size();
+ if (availableTaskCount <= 0) {
+ LOG.info("No available thread for disk balancer service. " +
+ "Current thread count is {}.", parallelThread);
+ return queue;
+ }
+
+ // TODO: Implementation for choose tasks
+
+ if (queue.isEmpty()) {
+ metrics.incrIdleLoopNoAvailableVolumePairCount();
+ }
+ return queue;
+ }
+
+ private boolean shouldDelay() {
+ // We should wait for next AvailableTime.
+ if (Time.monotonicNow() <= nextAvailableTime.get()) {
+ return true;
+ }
+ // Calculate the next AvailableTime based on bandwidth
+ long bytesBalanced = balancedBytesInLastWindow.getAndSet(0L);
+
+ final int megaByte = 1024 * 1024;
+
+ // converting disk bandwidth in byte/millisec
+ float bytesPerMillisec = bandwidthInMB * megaByte / 1000f;
+ nextAvailableTime.set(Time.monotonicNow() +
+ ((long) (bytesBalanced / bytesPerMillisec)));
+ return false;
+ }
+
+ private class DiskBalancerTask implements BackgroundTask {
+
+ private HddsVolume sourceVolume;
+ private HddsVolume destVolume;
+ private ContainerData containerData;
+
+ DiskBalancerTask(ContainerData containerData,
+ HddsVolume sourceVolume, HddsVolume destVolume) {
+ this.containerData = containerData;
+ this.sourceVolume = sourceVolume;
+ this.destVolume = destVolume;
+ }
+
+ @Override
+ public BackgroundTaskResult call() {
+ // TODO: Details of handling tasks
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+
+ @Override
+ public int getPriority() {
+ return BackgroundTask.super.getPriority();
+ }
+
+ private void postCall() {
+ inProgressContainers.remove(containerData.getContainerID());
+ deltaSizes.put(sourceVolume, deltaSizes.get(sourceVolume) +
+ containerData.getBytesUsed());
+ deltaSizes.put(destVolume, deltaSizes.get(destVolume)
+ - containerData.getBytesUsed());
+ }
+ }
+
+ private Path getDiskBalancerTmpDir(HddsVolume hddsVolume) {
+ return Paths.get(hddsVolume.getVolumeRootDir())
+ .resolve(DISK_BALANCER_TMP_DIR).resolve(DISK_BALANCER_DIR);
+ }
+
+ public boolean isBalancingContainer(long containerId) {
+ return inProgressContainers.contains(containerId);
+ }
+
+ public DiskBalancerServiceMetrics getMetrics() {
+ return metrics;
+ }
+
+ @VisibleForTesting
+ public void setBalancedBytesInLastWindow(long bytes) {
+ this.balancedBytesInLastWindow.set(bytes);
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ if (metrics != null) {
+ DiskBalancerServiceMetrics.unRegister();
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java
new file mode 100644
index 0000000000..3cf087b334
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+
+/**
+ * Metrics related to DiskBalancer Service running on Datanode.
+ */
+@Metrics(name = "DiskBalancerService Metrics", about = "Metrics related to "
+ + "background disk balancer service on Datanode", context = "dfs")
+public final class DiskBalancerServiceMetrics {
+
+ private static DiskBalancerServiceMetrics instance;
+ public static final String SOURCE_NAME =
+ DiskBalancerServiceMetrics.class.getSimpleName();
+
+ @Metric(about = "The number of successful balance job")
+ private MutableCounterLong successCount;
+
+ @Metric(about = "The total bytes for successfully balanced job.")
+ private MutableCounterLong successBytes;
+
+ @Metric(about = "The number of failed balance job.")
+ private MutableCounterLong failureCount;
+
+ @Metric(about = "The number of total running loop")
+ private MutableCounterLong runningLoopCount;
+
+ @Metric(about = "The number of idle loop that can not generate volume pair.")
+ private MutableCounterLong idleLoopNoAvailableVolumePairCount;
+
+ @Metric(about = "The number of idle loop due to bandwidth limits.")
+ private MutableCounterLong idleLoopExceedsBandwidthCount;
+
+ private DiskBalancerServiceMetrics() {
+ }
+
+ public static DiskBalancerServiceMetrics create() {
+ if (instance == null) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ instance = ms.register(SOURCE_NAME, "DiskBalancerService",
+ new DiskBalancerServiceMetrics());
+ }
+
+ return instance;
+ }
+
+ /**
+ * Unregister the metrics instance.
+ */
+ public static void unRegister() {
+ instance = null;
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ ms.unregisterSource(SOURCE_NAME);
+ }
+
+ public void incrSuccessCount(long count) {
+ this.successCount.incr(count);
+ }
+
+ public void incrSuccessBytes(long bytes) {
+ this.successBytes.incr(bytes);
+ }
+
+ public void incrFailureCount() {
+ this.failureCount.incr();
+ }
+
+ public void incrRunningLoopCount() {
+ this.runningLoopCount.incr();
+ }
+
+ public void incrIdleLoopNoAvailableVolumePairCount() {
+ this.idleLoopNoAvailableVolumePairCount.incr();
+ }
+
+ public void incrIdleLoopExceedsBandwidthCount() {
+ this.idleLoopExceedsBandwidthCount.incr();
+ }
+
+ public long getSuccessCount() {
+ return successCount.value();
+ }
+
+ public long getSuccessBytes() {
+ return successBytes.value();
+ }
+
+ public long getFailureCount() {
+ return failureCount.value();
+ }
+
+ public long getRunningLoopCount() {
+ return runningLoopCount.value();
+ }
+
+ public long getIdleLoopNoAvailableVolumePairCount() {
+ return idleLoopNoAvailableVolumePairCount.value();
+ }
+
+ public long getIdleLoopExceedsBandwidthCount() {
+ return idleLoopExceedsBandwidthCount.value();
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("successCount = " + successCount.value()).append("\t")
+ .append("successBytes = " + successBytes.value()).append("\t")
+ .append("failureCount = " + failureCount.value()).append("\t")
+ .append("idleLoopNoAvailableVolumePairCount = " +
+ idleLoopNoAvailableVolumePairCount.value()).append("\t")
+ .append("idleLoopExceedsBandwidthCount = " +
+ idleLoopExceedsBandwidthCount.value());
+ return buffer.toString();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java
new file mode 100644
index 0000000000..28da8993c1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Defines versions for the DiskBalancerService.
+ */
+public enum DiskBalancerVersion {
+ ONE(1, "First Version") {
+ };
+
+ private final int version;
+ private final String description;
+
+ public static final DiskBalancerVersion
+ DEFAULT_VERSION = DiskBalancerVersion.ONE;
+
+ private static final List<DiskBalancerVersion> DISK_BALANCER_VERSIONS =
+ ImmutableList.copyOf(values());
+
+ DiskBalancerVersion(int version, String description) {
+ this.version = version;
+ this.description = description;
+ }
+
+ public static DiskBalancerVersion getDiskBalancerVersion(int version) {
+ for (DiskBalancerVersion diskBalancerVersion :
+ DISK_BALANCER_VERSIONS) {
+ if (diskBalancerVersion.getVersion() == version) {
+ return diskBalancerVersion;
+ }
+ }
+ return null;
+ }
+
+ public static DiskBalancerVersion getDiskBalancerVersion(String versionStr) {
+ for (DiskBalancerVersion diskBalancerVersion :
+ DISK_BALANCER_VERSIONS) {
+ if (diskBalancerVersion.toString().equalsIgnoreCase(versionStr)) {
+ return diskBalancerVersion;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @return version number.
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * @return description.
+ */
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
new file mode 100644
index 0000000000..d16eb65747
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Class for creating diskbalancer.info file in yaml format.
+ */
+
+public final class DiskBalancerYaml {
+
+ private DiskBalancerYaml() {
+ // static helper methods only, no state.
+ }
+
+ /**
+ * Creates a yaml file to store DiskBalancer info.
+ *
+ * @param diskBalancerInfo {@link DiskBalancerInfo}
+ * @param path Path to diskBalancer.info file
+ */
+ public static void createDiskBalancerInfoFile(
+ DiskBalancerInfo diskBalancerInfo, File path)
+ throws IOException {
+ DumperOptions options = new DumperOptions();
+ options.setPrettyFlow(true);
+ options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW);
+ Yaml yaml = new Yaml(options);
+
+ try (Writer writer = new OutputStreamWriter(
+ new FileOutputStream(path), StandardCharsets.UTF_8)) {
+ yaml.dump(getDiskBalancerInfoYaml(diskBalancerInfo), writer);
+ }
+ }
+
+ /**
+ * Read DiskBalancerConfiguration from file.
+ */
+ public static DiskBalancerInfo readDiskBalancerInfoFile(File path)
+ throws IOException {
+ DiskBalancerInfo diskBalancerInfo;
+
+ try (FileInputStream inputFileStream = new FileInputStream(path)) {
+ Yaml yaml = new Yaml();
+ DiskBalancerInfoYaml diskBalancerInfoYaml;
+ try {
+ diskBalancerInfoYaml =
+ yaml.loadAs(inputFileStream, DiskBalancerInfoYaml.class);
+ } catch (Exception e) {
+ throw new IOException("Unable to parse yaml file.", e);
+ }
+
+ diskBalancerInfo = new DiskBalancerInfo(
+ diskBalancerInfoYaml.isShouldRun(),
+ diskBalancerInfoYaml.getThreshold(),
+ diskBalancerInfoYaml.getBandwidthInMB(),
+ diskBalancerInfoYaml.getParallelThread(),
+ DiskBalancerVersion.getDiskBalancerVersion(
+ diskBalancerInfoYaml.version));
+ }
+
+ return diskBalancerInfo;
+ }
+
+ /**
+ * Datanode DiskBalancer Info to be written to the yaml file.
+ */
+ public static class DiskBalancerInfoYaml {
+ private boolean shouldRun;
+ private double threshold;
+ private long bandwidthInMB;
+ private int parallelThread;
+
+ private int version;
+
+ public DiskBalancerInfoYaml() {
+ // Needed for snake-yaml introspection.
+ }
+
+ private DiskBalancerInfoYaml(boolean shouldRun, double threshold,
+ long bandwidthInMB, int parallelThread, int version) {
+ this.shouldRun = shouldRun;
+ this.threshold = threshold;
+ this.bandwidthInMB = bandwidthInMB;
+ this.parallelThread = parallelThread;
+ this.version = version;
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public void setBandwidthInMB(long bandwidthInMB) {
+ this.bandwidthInMB = bandwidthInMB;
+ }
+
+ public long getBandwidthInMB() {
+ return this.bandwidthInMB;
+ }
+
+ public void setParallelThread(int parallelThread) {
+ this.parallelThread = parallelThread;
+ }
+
+ public int getParallelThread() {
+ return this.parallelThread;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ public int getVersion() {
+ return this.version;
+ }
+ }
+
+ private static DiskBalancerInfoYaml getDiskBalancerInfoYaml(
+ DiskBalancerInfo diskBalancerInfo) {
+
+ return new DiskBalancerInfoYaml(
+ diskBalancerInfo.isShouldRun(),
+ diskBalancerInfo.getThreshold(),
+ diskBalancerInfo.getBandwidthInMB(),
+ diskBalancerInfo.getParallelThread(),
+ diskBalancerInfo.getVersion().getVersion());
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java
new file mode 100644
index 0000000000..d8ba71eb9d
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ This package contains classes related to the DiskBalancer service.
+ */
+package org.apache.hadoop.ozone.container.diskbalancer;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 6bb202e3cc..fb62874af8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
@@ -54,6 +55,8 @@ import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume.VolumeType;
import org.apache.hadoop.ozone.container.common.volume.StorageVolumeChecker;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerInfo;
+import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerService;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
import org.apache.hadoop.ozone.container.replication.ContainerImporter;
@@ -115,6 +118,7 @@ public class OzoneContainer {
private final StaleRecoveringContainerScrubbingService
recoveringContainerScrubbingService;
private final GrpcTlsConfig tlsClientConfig;
+ private final DiskBalancerService diskBalancerService;
private final AtomicReference<InitializingStatus> initializingStatus;
private final ReplicationServer replicationServer;
private DatanodeDetails datanodeDetails;
@@ -231,6 +235,15 @@ public class OzoneContainer {
blockDeletingServiceTimeout, TimeUnit.MILLISECONDS,
blockDeletingServiceWorkerSize, config);
+ Duration diskBalancerSvcInterval = conf.getObject(
+ DiskBalancerConfiguration.class).getDiskBalancerInterval();
+ Duration diskBalancerSvcTimeout = conf.getObject(
+ DiskBalancerConfiguration.class).getDiskBalancerTimeout();
+ diskBalancerService =
+ new DiskBalancerService(this, diskBalancerSvcInterval.toMillis(),
+ diskBalancerSvcTimeout.toMillis(), TimeUnit.MILLISECONDS, 1,
+ config);
+
Duration recoveringContainerScrubbingSvcInterval = conf.getObject(
DatanodeConfiguration.class).getRecoveringContainerScrubInterval();
@@ -410,6 +423,7 @@ public class OzoneContainer {
hddsDispatcher.init();
hddsDispatcher.setClusterId(clusterId);
blockDeletingService.start();
+ diskBalancerService.start();
recoveringContainerScrubbingService.start();
// mark OzoneContainer as INITIALIZED.
@@ -435,6 +449,7 @@ public class OzoneContainer {
dbVolumeSet.shutdown();
}
blockDeletingService.shutdown();
+ diskBalancerService.shutdown();
recoveringContainerScrubbingService.shutdown();
ContainerMetrics.remove();
}
@@ -531,7 +546,14 @@ public class OzoneContainer {
}
public DiskBalancerReportProto getDiskBalancerReport() {
- // TODO: Return real disk balancer report
- return null;
+ return diskBalancerService.getDiskBalancerReportProto();
+ }
+
+ public DiskBalancerInfo getDiskBalancerInfo() {
+ return diskBalancerService.getDiskBalancerInfo();
+ }
+
+ public DiskBalancerService getDiskBalancerService() {
+ return diskBalancerService;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
index ea97f34ca0..1780877787 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.protocol.commands;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
@@ -29,12 +30,12 @@ import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProt
*/
public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
- private final boolean shouldRun;
+ private final HddsProtos.DiskBalancerOpType opType;
private final DiskBalancerConfiguration diskBalancerConfiguration;
- public DiskBalancerCommand(final boolean shouldRun,
+ public DiskBalancerCommand(final HddsProtos.DiskBalancerOpType opType,
final DiskBalancerConfiguration diskBalancerConfiguration) {
- this.shouldRun = shouldRun;
+ this.opType = opType;
this.diskBalancerConfiguration = diskBalancerConfiguration;
}
@@ -50,22 +51,26 @@ public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
@Override
public DiskBalancerCommandProto getProto() {
- return DiskBalancerCommandProto.newBuilder()
- .setShouldRun(shouldRun)
- .setDiskBalancerConf(diskBalancerConfiguration.toProtobufBuilder())
- .build();
+ DiskBalancerCommandProto.Builder builder = DiskBalancerCommandProto
+ .newBuilder().setOpType(opType);
+ // Stop command don't have diskBalancerConf
+ if (diskBalancerConfiguration != null) {
+ builder.setDiskBalancerConf(
+ diskBalancerConfiguration.toProtobufBuilder());
+ }
+ return builder.build();
}
public static DiskBalancerCommand getFromProtobuf(DiskBalancerCommandProto
diskbalancerCommandProto, ConfigurationSource configuration) {
Preconditions.checkNotNull(diskbalancerCommandProto);
- return new DiskBalancerCommand(diskbalancerCommandProto.getShouldRun(),
+ return new DiskBalancerCommand(diskbalancerCommandProto.getOpType(),
DiskBalancerConfiguration.fromProtobuf(
diskbalancerCommandProto.getDiskBalancerConf(), configuration));
}
- public boolean isShouldRun() {
- return shouldRun;
+ public HddsProtos.DiskBalancerOpType getOpType() {
+ return opType;
}
public DiskBalancerConfiguration getDiskBalancerConfiguration() {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
new file mode 100644
index 0000000000..0d87ac3420
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceTestImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A test class implementation for {@link DiskBalancerService}.
+ */
+public class DiskBalancerServiceTestImpl extends DiskBalancerService {
+
+ // the service timeout
+ private static final int SERVICE_TIMEOUT_IN_MILLISECONDS = 0;
+
+ // tests only
+ private CountDownLatch latch;
+ private Thread testingThread;
+ private AtomicInteger numOfProcessed = new AtomicInteger(0);
+
+ public DiskBalancerServiceTestImpl(OzoneContainer container,
+ int serviceInterval, ConfigurationSource conf, int threadCount)
+ throws IOException {
+ super(container, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+ TimeUnit.MILLISECONDS, threadCount, conf);
+ }
+
+ public void runBalanceTasks() {
+ if (latch.getCount() > 0) {
+ this.latch.countDown();
+ } else {
+ throw new IllegalStateException("Count already reaches zero");
+ }
+ }
+
+ public boolean isStarted() {
+ return latch != null && testingThread.isAlive();
+ }
+
+ public int getTimesOfProcessed() {
+ return numOfProcessed.get();
+ }
+
+ // Override the implementation to start a single on-call control thread.
+ @Override
+ public void start() {
+ PeriodicalTask svc = new PeriodicalTask();
+ // In test mode, relies on a latch countdown to runDeletingTasks tasks.
+ Runnable r = () -> {
+ while (true) {
+ latch = new CountDownLatch(1);
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ break;
+ }
+ Future<?> future = this.getExecutorService().submit(svc);
+ try {
+ // for tests, we only wait for 3s for completion
+ future.get(3000, TimeUnit.SECONDS);
+ numOfProcessed.incrementAndGet();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return;
+ }
+ }
+ };
+
+ testingThread = new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .build()
+ .newThread(r);
+ testingThread.start();
+ }
+
+ @Override
+ public void shutdown() {
+ testingThread.interrupt();
+ super.shutdown();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
new file mode 100644
index 0000000000..a58f00103b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.container.common.TestBlockDeletingService;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
+import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mock;
+
+/**
+ * This is a test class for DiskBalancerService.
+ */
+@RunWith(Parameterized.class)
+public class TestDiskBalancerService {
+ private File testRoot;
+ private String scmId;
+ private String datanodeUuid;
+ private OzoneConfiguration conf;
+
+ private final ContainerLayoutVersion layout;
+ private final String schemaVersion;
+ private MutableVolumeSet volumeSet;
+
+ public TestDiskBalancerService(ContainerTestVersionInfo versionInfo) {
+ this.layout = versionInfo.getLayout();
+ this.schemaVersion = versionInfo.getSchemaVersion();
+ conf = new OzoneConfiguration();
+ ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
+ }
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> parameters() {
+ return ContainerTestVersionInfo.versionParameters();
+ }
+
+ @Before
+ public void init() throws IOException {
+ testRoot = GenericTestUtils
+ .getTestDir(TestBlockDeletingService.class.getSimpleName());
+ if (testRoot.exists()) {
+ FileUtils.cleanDirectory(testRoot);
+ }
+ scmId = UUID.randomUUID().toString();
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY,
+ generateVolumeLocation(testRoot.getAbsolutePath(), 2));
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testRoot.getAbsolutePath());
+ conf.set("hdds.datanode.du.factory.classname",
+ "org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory$HalfTera");
+ datanodeUuid = UUID.randomUUID().toString();
+ volumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ createDbInstancesForTestIfNeeded(volumeSet, scmId, scmId, conf);
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ BlockUtils.shutdownCache(conf);
+ FileUtils.deleteDirectory(testRoot);
+ }
+
+ @Test
+ @Timeout(30)
+ public void testUpdateService() throws Exception {
+ // Increase volume's usedBytes
+ for (StorageVolume volume : volumeSet.getVolumeMap().values()) {
+ volume.incrementUsedSpace(volume.getCapacity() / 2);
+ }
+
+ ContainerSet containerSet = new ContainerSet(1000);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ KeyValueHandler keyValueHandler =
+ new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
+ metrics, c -> {
+ });
+ DiskBalancerServiceTestImpl svc =
+ getDiskBalancerService(containerSet, conf, keyValueHandler, null, 1);
+
+ // Set a low bandwidth to delay job
+ svc.setShouldRun(true);
+ svc.setThreshold(10.0d);
+ svc.setBandwidthInMB(1L);
+ svc.setParallelThread(5);
+ svc.setVersion(DiskBalancerVersion.DEFAULT_VERSION);
+
+ svc.start();
+
+ Assert.assertTrue(svc.getDiskBalancerInfo().isShouldRun());
+ Assert.assertEquals(10, svc.getDiskBalancerInfo().getThreshold(), 0.0);
+ Assert.assertEquals(1, svc.getDiskBalancerInfo().getBandwidthInMB());
+ Assert.assertEquals(5, svc.getDiskBalancerInfo().getParallelThread());
+
+ DiskBalancerInfo newInfo = new DiskBalancerInfo(false, 20.0d, 5L, 10);
+ svc.refresh(newInfo);
+
+ Assert.assertFalse(svc.getDiskBalancerInfo().isShouldRun());
+ Assert.assertEquals(20, svc.getDiskBalancerInfo().getThreshold(), 0.0);
+ Assert.assertEquals(5, svc.getDiskBalancerInfo().getBandwidthInMB());
+ Assert.assertEquals(10, svc.getDiskBalancerInfo().getParallelThread());
+
+ svc.shutdown();
+ }
+
+ private String generateVolumeLocation(String base, int volumeCount) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < volumeCount; i++) {
+ sb.append(base + "/vol" + i);
+ sb.append(",");
+ }
+ return sb.substring(0, sb.length() - 1);
+ }
+
+ private DiskBalancerServiceTestImpl getDiskBalancerService(
+ ContainerSet containerSet, ConfigurationSource config,
+ KeyValueHandler keyValueHandler, ContainerController controller,
+ int threadCount) throws IOException {
+ OzoneContainer ozoneContainer =
+ mockDependencies(containerSet, keyValueHandler, controller);
+ return new DiskBalancerServiceTestImpl(ozoneContainer, 1000, config,
+ threadCount);
+ }
+
+ private OzoneContainer mockDependencies(ContainerSet containerSet,
+ KeyValueHandler keyValueHandler, ContainerController controller) {
+ OzoneContainer ozoneContainer = mock(OzoneContainer.class);
+ when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
+ when(ozoneContainer.getWriteChannel()).thenReturn(null);
+ ContainerDispatcher dispatcher = mock(ContainerDispatcher.class);
+ when(ozoneContainer.getDispatcher()).thenReturn(dispatcher);
+ when(dispatcher.getHandler(any())).thenReturn(keyValueHandler);
+ when(ozoneContainer.getVolumeSet()).thenReturn(volumeSet);
+ when(ozoneContainer.getController()).thenReturn(controller);
+ return ozoneContainer;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
new file mode 100644
index 0000000000..3b61255b0b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.diskbalancer;
+
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT;
+
+/**
+ * Tests to test DiskBalancer's YAML operation.
+ */
+public class TestDiskBalancerYaml {
+ @Test
+ public void testCreateYaml() throws IOException {
+ boolean shouldRun = true;
+ double threshold = 10;
+ long bandwidthInMB = 100;
+ int parallelThread = 5;
+ DiskBalancerVersion version = DiskBalancerVersion.DEFAULT_VERSION;
+
+ File file = new File(GenericTestUtils.getTestDir(),
+ OZONE_SCM_DATANODE_DISK_BALANCER_INFO_FILE_DEFAULT);
+
+ DiskBalancerInfo info = new DiskBalancerInfo(shouldRun, threshold,
+ bandwidthInMB, parallelThread, version);
+
+ DiskBalancerYaml.createDiskBalancerInfoFile(info, file);
+
+ DiskBalancerInfo newInfo = DiskBalancerYaml.readDiskBalancerInfoFile(file);
+
+ Assertions.assertEquals(info, newInfo);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 0e177449d1..d9fb97737e 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
@@ -1114,7 +1113,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.start)
+ .setOpType(HddsProtos.DiskBalancerOpType.START)
.setConf(confBuilder);
hosts.ifPresent(requestBuilder::addAllHosts);
@@ -1136,7 +1135,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
throws IOException {
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.stop);
+ .setOpType(HddsProtos.DiskBalancerOpType.STOP);
hosts.ifPresent(requestBuilder::addAllHosts);
DatanodeDiskBalancerOpResponseProto response =
@@ -1165,7 +1164,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
DatanodeDiskBalancerOpRequestProto.newBuilder()
- .setOpType(DatanodeDiskBalancerOpType.update)
+ .setOpType(HddsProtos.DiskBalancerOpType.UPDATE)
.setConf(confBuilder);
hosts.ifPresent(requestBuilder::addAllHosts);
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index e6f5fbde79..79b1a5d18f 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -357,14 +357,8 @@ message DatanodeDiskBalancerInfoResponseProto {
repeated DatanodeDiskBalancerInfoProto info = 1;
}
-enum DatanodeDiskBalancerOpType{
- start = 1;
- stop = 2;
- update = 3;
-}
-
message DatanodeDiskBalancerOpRequestProto {
- required DatanodeDiskBalancerOpType opType = 1;
+ required DiskBalancerOpType opType = 1;
repeated string hosts = 2;
optional DiskBalancerConfigurationProto conf = 3;
}
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 4cf60338d2..083d948fee 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -479,6 +479,12 @@ enum DiskBalancerRunningStatus {
UNKNOWN = 3;
}
+enum DiskBalancerOpType{
+ START = 1;
+ STOP = 2;
+ UPDATE = 3;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 81e9ee54d3..f7893f2d60 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -449,7 +449,7 @@ message DatanodeDetailsAndReplicaIndexProto {
This command asks the datanode to update diskBalancer status
*/
message DiskBalancerCommandProto {
- required bool shouldRun = 1;
+ required DiskBalancerOpType opType = 1;
optional DiskBalancerConfigurationProto diskBalancerConf = 2;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index c6114c52f5..1af9bfde4c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -24,10 +24,15 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
+import org.apache.ratis.protocol.exceptions.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -126,6 +131,118 @@ public class DiskBalancerManager {
}
}
+ /**
+ * Send startDiskBalancer command to datanodes.
+ * If hosts is not specified, send commands to all healthy datanodes.
+ * @param threshold new configuration of threshold
+ * @param bandwidthInMB new configuration of bandwidthInMB
+ * @param parallelThread new configuration of parallelThread
+ * @param hosts Datanodes that command will apply on
+ * @return Possible errors
+ * @throws IOException
+ */
+ public List<DatanodeAdminError> startDiskBalancer(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ if (nodeManager.getNodeStatus(dn).isHealthy()) {
+ errors.add(new DatanodeAdminError(dn.getHostName(),
+ "Datanode not in healthy state"));
+ continue;
+ }
+ // If command doesn't have configuration change, then we reuse the
+ // latest configuration reported from Datnaodes
+ DiskBalancerConfiguration updateConf = attachDiskBalancerConf(dn,
+ threshold, bandwidthInMB, parallelThread);
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.START, updateConf);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
+ /**
+ * Send stopDiskBalancer command to datanodes
+ * If hosts is not specified, send commands to all datanodes.
+ * @param hosts Datanodes that command will apply on
+ * */
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.STOP, null);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
+
+ /**
+ * Send update DiskBalancerConf command to datanodes.
+ * If hosts is not specified, send commands to all healthy datanodes.
+ * @param threshold new configuration of threshold
+ * @param bandwidthInMB new configuration of bandwidthInMB
+ * @param parallelThread new configuration of parallelThread
+ * @param hosts Datanodes that command will apply on
+ * @return Possible errors
+ * @throws IOException
+ */
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
+ throws IOException {
+ List<DatanodeDetails> dns;
+ if (hosts.isPresent()) {
+ dns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ } else {
+ dns = nodeManager.getNodes(NodeStatus.inServiceHealthy());
+ }
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeDetails dn : dns) {
+ try {
+ // If command doesn't have configuration change, then we reuse the
+ // latest configuration reported from Datnaodes
+ DiskBalancerConfiguration updateConf = attachDiskBalancerConf(dn,
+ threshold, bandwidthInMB, parallelThread);
+ DiskBalancerCommand command = new DiskBalancerCommand(
+ HddsProtos.DiskBalancerOpType.UPDATE, updateConf);
+ sendCommand(dn, command);
+ } catch (Exception e) {
+ errors.add(new DatanodeAdminError(dn.getHostName(), e.getMessage()));
+ }
+ }
+ return errors;
+ }
+
private boolean shouldReturnDatanode(
HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
@@ -218,6 +335,30 @@ public class DiskBalancerManager {
}
}
+ private DiskBalancerConfiguration attachDiskBalancerConf(
+ DatanodeDetails dn, Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread) {
+ DiskBalancerConfiguration baseConf = statusMap.containsKey(dn) ?
+ statusMap.get(dn).getDiskBalancerConfiguration() :
+ new DiskBalancerConfiguration();
+ threshold.ifPresent(baseConf::setThreshold);
+ bandwidthInMB.ifPresent(baseConf::setDiskBandwidthInMB);
+ parallelThread.ifPresent(baseConf::setParallelThread);
+ return baseConf;
+ }
+
+ private void sendCommand(DatanodeDetails dn, DiskBalancerCommand command) {
+ try {
+ command.setTerm(scmContext.getTermOfLeader());
+ } catch (NotLeaderException nle) {
+ LOG.warn("Skip sending DiskBalancerCommand for Datanode {}," +
+ " since not leader SCM.", dn.getUuidString());
+ return;
+ }
+ scmNodeEventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+ new CommandForDatanode<>(dn.getUuid(), command));
+ }
+
@VisibleForTesting
public Map<DatanodeDetails, DiskBalancerStatus> getStatusMap() {
return statusMap;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index c656cd9ffb..2c6779ab56 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -1258,21 +1258,21 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
throws IOException {
List<DatanodeAdminError> errors;
switch (request.getOpType()) {
- case start:
+ case START:
errors = impl.startDiskBalancer(
Optional.of(request.getConf().getThreshold()),
Optional.of(request.getConf().getDiskBandwidthInMB()),
Optional.of(request.getConf().getParallelThread()),
Optional.of(request.getHostsList()));
break;
- case update:
+ case UPDATE:
errors = impl.updateDiskBalancerConfiguration(
Optional.of(request.getConf().getThreshold()),
Optional.of(request.getConf().getDiskBandwidthInMB()),
Optional.of(request.getConf().getParallelThread()),
Optional.of(request.getHostsList()));
break;
- case stop:
+ case STOP:
errors = impl.stopDiskBalancer(Optional.of(request.getHostsList()));
break;
default:
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 3fd3d8d0fb..2bdc82609b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1212,6 +1212,7 @@ public class SCMClientProtocolServer implements
ContainerID.valueOf(startContainerID), count, state);
}
+ @Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
int count, int clientVersion) throws IOException {
// check admin authorisation
@@ -1247,15 +1248,27 @@ public class SCMClientProtocolServer implements
public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager()
+ .startDiskBalancer(threshold, bandwidthInMB, parallelThread, hosts);
}
@Override
public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+ return scm.getDiskBalancerManager().stopDiskBalancer(hosts);
}
@@ -1264,8 +1277,15 @@ public class SCMClientProtocolServer implements
Optional<Double> threshold, Optional<Long> bandwidthInMB,
Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
- // TODO: Send message to datanodes
- return new ArrayList<DatanodeAdminError>();
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().updateDiskBalancerConfiguration(
+ threshold, bandwidthInMB, parallelThread, hosts);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 04/05: HDDS-7234. Add a common option for DiskBalancer commands (#3762)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1485590510121895b1b67dfb9765f0a2b08534d9
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 22 10:00:39 2022 +0800
HDDS-7234. Add a common option for DiskBalancer commands (#3762)
---
.../scm/cli/datanode/DiskBalancerCommands.java | 18 ++---
.../cli/datanode/DiskBalancerCommonOptions.java | 84 ++++++++++++++++++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 27 +++----
.../cli/datanode/DiskBalancerStatusSubcommand.java | 7 +-
.../cli/datanode/DiskBalancerStopSubcommand.java | 29 +++-----
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 27 +++----
6 files changed, 123 insertions(+), 69 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
index 649ee59eaa..dd70bbb491 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -38,36 +38,38 @@ import java.util.concurrent.Callable;
* [ -t/--threshold {@literal <threshold>}]
* [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
* [ -p/--parallelThread {@literal <parallelThread>}]
- * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ -a/--all {@literal <alldatanodes>}]
+ * [ -d/--datanodes {@literal <datanodes>}]
* [ {@literal <hosts>}]
* Examples:
- * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -d {@literal <hosts>}
* start balancer with default values in the configuration on specified
* datanodes
* ozone admin datanode diskbalancer start -a
* start balancer with default values in the configuration on all
* datanodes in the cluster
- * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -t 5 -d {@literal <hosts>}
* start balancer with a threshold of 5%
- * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -b 20 -d {@literal <hosts>}
* start balancer with maximum 20MB/s diskbandwidth
- * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -p 5 -d {@literal <hosts>}
* start balancer with 5 parallel thread on each datanode
* To stop:
* ozone admin datanode diskbalancer stop -a
* stop diskblancer on all datanodes
- * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * ozone admin datanode diskbalancer stop -d {@literal <hosts>};
* stop diskblancer on all datanodes
* To update:
* ozone admin datanode diskbalancer update -a
* update diskblancer configuration on all datanodes
- * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * ozone admin datanode diskbalancer update -d {@literal <hosts>};
* update diskblancer configuration on all datanodes
* To get report:
* ozone admin datanode diskbalancer report -c 10
* retrieve at most 10 datanodes that needs diskbalance most
* To get status:
- * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * ozone admin datanode diskbalancer status -s RUNNING -d
+ * {@literal <hosts>}
* return the diskbalancer status on datanodes where diskbalancer are in
* Running state
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
new file mode 100644
index 0000000000..744de5169f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import picocli.CommandLine;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Common options for DiskBalancer commands.
+ */
+public class DiskBalancerCommonOptions {
+
+ @CommandLine.Option(names = {"-a", "--all"},
+ description = "Run commands on all datanodes.")
+ private boolean allHosts;
+
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Run commands on specific datanodes, the content can be " +
+ "a list of hostnames or IPs. " +
+ "Examples: hostname1,hostname2,hostname3 or ip1,ip2,ip3")
+ private List<String> datanodes = new ArrayList<>();
+
+ /**
+ * Check the common options of DiskBalancerCommand.
+ * @return if the check passed
+ */
+ public boolean check() {
+ if (datanodes.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify at least " +
+ "one datanode or use \"-a(--all)\" to start diskBalancer " +
+ "on all datanodes");
+ return false;
+ }
+ if (datanodes.size() != 0 && allHosts) {
+ System.out.println("Invalid option selection. " +
+ "Use either \"-a(--all)\" or \"-d(--datanodes)\".");
+ return false;
+ }
+ return true;
+ }
+
+ public String getHostString() {
+ return isAllHosts() ? "All datanodes" : String.join("\n", getDatanodes());
+ }
+
+ public Optional<List<String>> getSpecifiedDatanodes() {
+ return getDatanodes().size() == 0 ?
+ Optional.empty() : Optional.of(getDatanodes());
+ }
+
+ public boolean isAllHosts() {
+ return allHosts;
+ }
+
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+
+ public List<String> getDatanodes() {
+ return datanodes;
+ }
+
+ public void setDatanodes(List<String> datanodes) {
+ this.datanodes = datanodes;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
index e3862ba23f..f58c7a1f84 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified. Please specify " +
- "\"--allDatanodes\" to start diskBalancer on all datanodes");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Start DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -95,6 +84,6 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
index 6a61876479..84dd3abe95 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -21,9 +21,9 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
import java.util.ArrayList;
@@ -48,10 +48,11 @@ public class DiskBalancerStatusSubcommand extends ScmSubcommand {
new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
@Option(names = {"-s", "--state"},
- description = "RUNNING, STOPPED, UNKNOWN")
+ description = "RUNNING, STOPPED, UNKNOWN. Default state is RUNNING.")
private String state = "RUNNING";
- @Parameters(description = "List of fully qualified host names")
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Get diskBalancer status on specific datanodes.")
private List<String> hosts = new ArrayList<>();
@Override
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
index 9479c030a6..f63f880a44 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -22,14 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
/**
* Handler to stop disk balancer.
@@ -40,29 +37,21 @@ import java.util.Optional;
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class DiskBalancerStopSubcommand extends ScmSubcommand {
- @Option(names = {"-a", "--allDatanodes"},
- description = "Stop diskBalancer on all datanodes.")
- private boolean allHosts;
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
+ if (!commonOptions.check()) {
return;
}
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
- return;
- }
- List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
- Optional.empty() : Optional.of(hosts));
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Stopping DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
System.err.println("Error: " + error.getHostname() + ": "
@@ -77,6 +66,6 @@ public class DiskBalancerStopSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
index b19fb6a53a..28ed4fd325 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
- parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ parallelThread, commonOptions.getSpecifiedDatanodes());
System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -93,6 +82,6 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/05: HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 248af791814d6148b98a1a8cb52ca3f4b8451fec
Author: Symious <yi...@foxmail.com>
AuthorDate: Wed Sep 7 12:19:57 2022 +0800
HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 21 ++--
.../protocol/StorageContainerLocationProtocol.java | 17 ++--
.../scm/storage/DiskBalancerConfiguration.java | 39 +++----
.../common/src/main/resources/ozone-default.xml | 8 ++
.../common/report/DiskBalancerReportPublisher.java | 66 ++++++++++++
.../common/report/ReportPublisherFactory.java | 3 +
.../common/statemachine/DatanodeStateMachine.java | 2 +
.../common/statemachine/StateContext.java | 13 +++
.../commandhandler/DiskBalancerCommandHandler.java | 112 +++++++++++++++++++++
.../states/endpoint/HeartbeatEndpointTask.java | 7 ++
.../ozone/container/ozoneimpl/OzoneContainer.java | 6 ++
.../protocol/commands/DiskBalancerCommand.java | 74 ++++++++++++++
.../common/report/TestReportPublisher.java | 42 ++++++++
.../common/report/TestReportPublisherFactory.java | 12 +++
...inerLocationProtocolClientSideTranslatorPB.java | 81 +++++++++++++--
.../src/main/proto/ScmAdminProtocol.proto | 24 ++++-
.../interface-client/src/main/proto/hdds.proto | 10 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 17 ++++
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 9 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 111 ++++++++++++++------
.../hdds/scm/node/DiskBalancerReportHandler.java | 65 ++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 2 -
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 17 ++++
...inerLocationProtocolServerSideTranslatorPB.java | 51 +++++++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 22 ++--
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 23 +++++
.../hdds/scm/server/StorageContainerManager.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 51 +++++++++-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../hdds/scm/node/TestDiskBalancerManager.java | 34 +++++++
.../testutils/ReplicationNodeManagerMock.java | 6 ++
.../hdds/scm/cli/ContainerOperationClient.java | 29 +++---
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +
35 files changed, 901 insertions(+), 100 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index cb258dfa74..7b3a9c0dd7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -46,6 +46,10 @@ public final class HddsConfigKeys {
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
"60s";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL =
+ "hdds.disk.balancer.report.interval";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT =
+ "60s";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index d4f6267c49..eb62b66c8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -422,7 +422,8 @@ public interface ScmClient extends Closeable {
throws IOException;
/**
- * Get DiskBalancer status.
+ * Get DiskBalancer report.
+ * REPORT shows the current volume density of datanodes.
* @param count top datanodes that need balancing
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
@@ -432,33 +433,39 @@ public interface ScmClient extends Closeable {
/**
* Get DiskBalancer status.
+ * STATUS shows the running status of DiskBalancer on datanodes.
* @param hosts If hosts is not null, return status of hosts; If hosts is
* null, return status of all datanodes in balancing.
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidth,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index d3e02a5104..7c0317953c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -463,26 +463,31 @@ public interface StorageContainerLocationProtocol extends Closeable {
* Get DiskBalancer status.
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus,
+ int clientVersion) throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index 704b383679..43417f2e81 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
@@ -36,7 +36,7 @@ public final class DiskBalancerConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerConfiguration.class);
- @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ @Config(key = "volume.density.threshold", type = ConfigType.DOUBLE,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "Threshold is a percentage in the range of 0 to 100. A " +
"datanode is considered balanced if for each volume, the " +
@@ -45,12 +45,12 @@ public final class DiskBalancerConfiguration {
" of the entire datanode) no more than the threshold.")
private double threshold = 10d;
- @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.LONG,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "The max balance speed.")
- private double diskBandwidth = 10;
+ private long diskBandwidthInMB = 10;
- @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ @Config(key = "parallel.thread", type = ConfigType.INT,
defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
description = "The max parallel balance thread count.")
private int parallelThread = 5;
@@ -86,21 +86,21 @@ public final class DiskBalancerConfiguration {
*
* @return max disk bandwidth per second
*/
- public double getDiskBandwidth() {
- return diskBandwidth;
+ public double getDiskBandwidthInMB() {
+ return diskBandwidthInMB;
}
/**
* Sets the disk bandwidth value for Disk Balancer.
*
- * @param diskBandwidth the bandwidth to control balance speed
+ * @param diskBandwidthInMB the bandwidth to control balance speed
*/
- public void setDiskBandwidth(double diskBandwidth) {
- if (diskBandwidth <= 0d) {
+ public void setDiskBandwidthInMB(long diskBandwidthInMB) {
+ if (diskBandwidthInMB <= 0L) {
throw new IllegalArgumentException(
- "diskBandwidth must be a value larger than 0.");
+ "diskBandwidthInMB must be a value larger than 0.");
}
- this.diskBandwidth = diskBandwidth;
+ this.diskBandwidthInMB = diskBandwidthInMB;
}
/**
@@ -124,6 +124,7 @@ public final class DiskBalancerConfiguration {
}
this.parallelThread = parallelThread;
}
+
@Override
public String toString() {
return String.format("Disk Balancer Configuration values:%n" +
@@ -132,7 +133,7 @@ public final class DiskBalancerConfiguration {
"%-50s %s%n" +
"%-50s %s%n",
"Key", "Value",
- "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidthInMB,
"Parallel Thread", parallelThread);
}
@@ -141,21 +142,21 @@ public final class DiskBalancerConfiguration {
HddsProtos.DiskBalancerConfigurationProto.newBuilder();
builder.setThreshold(threshold)
- .setDiskBandwidth(diskBandwidth)
+ .setDiskBandwidthInMB(diskBandwidthInMB)
.setParallelThread(parallelThread);
return builder;
}
- static DiskBalancerConfiguration fromProtobuf(
+ public static DiskBalancerConfiguration fromProtobuf(
@NotNull HddsProtos.DiskBalancerConfigurationProto proto,
- @NotNull OzoneConfiguration ozoneConfiguration) {
+ @NotNull ConfigurationSource configurationSource) {
DiskBalancerConfiguration config =
- ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ configurationSource.getObject(DiskBalancerConfiguration.class);
if (proto.hasThreshold()) {
config.setThreshold(proto.getThreshold());
}
- if (proto.hasDiskBandwidth()) {
- config.setDiskBandwidth(proto.getDiskBandwidth());
+ if (proto.hasDiskBandwidthInMB()) {
+ config.setDiskBandwidthInMB(proto.getDiskBandwidthInMB());
}
if (proto.hasParallelThread()) {
config.setParallelThread(proto.getParallelThread());
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 8c8ae634e2..e6dc3d46da 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -363,6 +363,14 @@
datanode periodically send pipeline report to SCM. Unit could be
defined with postfix (ns,ms,s,m,h,d)</description>
</property>
+ <property>
+ <name>hdds.disk.balancer.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, CONTAINER, DISK_BALANCER</tag>
+ <description>Time interval of the datanode to send disk balancer report. Each
+ datanode periodically sends disk balancer report to SCM. Unit could be
+ defined with postfix (ns,ms,s,m,h,d)</description>
+ </property>
<property>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
new file mode 100644
index 0000000000..2bb78c1ee6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.report;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT;
+
+
+/**
+ * Publishes DiskBalancer report which will be sent to SCM as part of heartbeat.
+ * DiskBalancer Report consist of the following information:
+ * - isBalancerRunning
+ * - balancedBytes
+ * - DiskBalancerConfiguration
+ */
+public class DiskBalancerReportPublisher extends
+ ReportPublisher<DiskBalancerReportProto> {
+
+ private Long diskBalancerReportInterval = null;
+
+ @Override
+ protected long getReportFrequency() {
+ if (diskBalancerReportInterval == null) {
+ diskBalancerReportInterval = getConf().getTimeDuration(
+ HDDS_DISK_BALANCER_REPORT_INTERVAL,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+ getConf());
+
+ Preconditions.checkState(
+ heartbeatFrequency <= diskBalancerReportInterval,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL +
+ " cannot be configured lower than heartbeat frequency " +
+ heartbeatFrequency + ".");
+ }
+ return diskBalancerReportInterval;
+ }
+
+ @Override
+ protected DiskBalancerReportProto getReport() {
+ return getContext().getParent().getContainer().getDiskBalancerReport();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index 3be1b5e077..19d2806919 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -55,6 +56,8 @@ public class ReportPublisherFactory {
report2publisher.put(PipelineReportsProto.class,
PipelineReportPublisher.class);
report2publisher.put(CRLStatusReport.class, CRLStatusReportPublisher.class);
+ report2publisher.put(DiskBalancerReportProto.class,
+ DiskBalancerReportPublisher.class);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 5536b02e54..f9cceb6191 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.Comm
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CreatePipelineCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DiskBalancerCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.FinalizeNewLayoutVersionCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ReconstructECContainersCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.RefreshVolumeUsageCommandHandler;
@@ -228,6 +229,7 @@ public class DatanodeStateMachine implements Closeable {
.addHandler(new SetNodeOperationalStateCommandHandler(conf))
.addHandler(new FinalizeNewLayoutVersionCommandHandler())
.addHandler(new RefreshVolumeUsageCommandHandler())
+ .addHandler(new DiskBalancerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index c4d3428a80..c75c7a455d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
@@ -98,6 +99,9 @@ public class StateContext {
@VisibleForTesting
static final String CRL_STATUS_REPORT_PROTO_NAME =
CRLStatusReport.getDescriptor().getFullName();
+ @VisibleForTesting
+ static final String DISK_BALANCER_REPORT_PROTO_NAME =
+ DiskBalancerReportProto.getDescriptor().getFullName();
static final Logger LOG =
LoggerFactory.getLogger(StateContext.class);
@@ -113,6 +117,7 @@ public class StateContext {
private final AtomicReference<Message> nodeReport;
private final AtomicReference<Message> pipelineReports;
private final AtomicReference<Message> crlStatusReport;
+ private final AtomicReference<Message> diskBalancerReport;
// Incremental reports are queued in the map below
private final Map<InetSocketAddress, List<Message>>
incrementalReportsQueue;
@@ -172,6 +177,7 @@ public class StateContext {
nodeReport = new AtomicReference<>();
pipelineReports = new AtomicReference<>();
crlStatusReport = new AtomicReference<>(); // Certificate Revocation List
+ diskBalancerReport = new AtomicReference<>();
endpoints = new HashSet<>();
containerActions = new HashMap<>();
pipelineActions = new HashMap<>();
@@ -197,6 +203,8 @@ public class StateContext {
type2Reports.put(PIPELINE_REPORTS_PROTO_NAME, pipelineReports);
fullReportTypeList.add(CRL_STATUS_REPORT_PROTO_NAME);
type2Reports.put(CRL_STATUS_REPORT_PROTO_NAME, crlStatusReport);
+ fullReportTypeList.add(DISK_BALANCER_REPORT_PROTO_NAME);
+ type2Reports.put(DISK_BALANCER_REPORT_PROTO_NAME, diskBalancerReport);
}
/**
@@ -929,6 +937,11 @@ public class StateContext {
return crlStatusReport.get();
}
+ @VisibleForTesting
+ public Message getDiskBalancerReport() {
+ return diskBalancerReport.get();
+ }
+
public void configureReconHeartbeatFrequency() {
reconHeartbeatFrequency.set(getReconHeartbeatInterval(conf));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
new file mode 100644
index 0000000000..705c0af0e1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Handler for DiskBalancer command received from SCM.
+ */
+public class DiskBalancerCommandHandler implements CommandHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerCommandHandler.class);
+
+ private AtomicLong invocationCount = new AtomicLong(0);
+ private long totalTime;
+
+ /**
+ * Constructs a diskBalancerCommand handler.
+ */
+ public DiskBalancerCommandHandler() {
+ }
+
+ /**
+ * Handles a given SCM command.
+ *
+ * @param command - SCM Command
+ * @param ozoneContainer - Ozone Container.
+ * @param context - Current Context.
+ * @param connectionManager - The SCMs that we are talking to.
+ */
+ @Override
+ public void handle(SCMCommand command, OzoneContainer ozoneContainer,
+ StateContext context, SCMConnectionManager connectionManager) {
+ invocationCount.incrementAndGet();
+ // TODO: Do start/stop/update operation
+ }
+
+ /**
+ * Returns the command type that this command handler handles.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getCommandType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ /**
+ * Returns number of times this handler has been invoked.
+ *
+ * @return int
+ */
+ @Override
+ public int getInvocationCount() {
+ return (int)invocationCount.get();
+ }
+
+ /**
+ * Returns the average time this function takes to run.
+ *
+ * @return long
+ */
+ @Override
+ public long getAverageRunTime() {
+ if (invocationCount.get() > 0) {
+ return totalTime / invocationCount.get();
+ }
+ return 0;
+ }
+
+ @Override
+ public int getQueuedCount() {
+ return 0;
+ }
+
+ private void startDiskBalancer(DiskBalancerConfiguration configuration) {
+ // Todo: add implementation to start DiskBalancer
+ }
+
+ private void stopDiskBalancer() {
+ // Todo: add implementation to stop DiskBalancer
+ }
+
+ private void updateDiskBalancer(DiskBalancerConfiguration
+ configuration) {
+ // Todo: add implementation to update diskBalancer configuration
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 3a1bd8ffb3..ac5a14b6aa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
import org.apache.hadoop.ozone.protocol.commands.FinalizeNewLayoutVersionCommand;
import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
import org.apache.hadoop.ozone.protocol.commands.RefreshVolumeUsageCommand;
@@ -418,6 +419,12 @@ public class HeartbeatEndpointTask
commandResponseProto.getRefreshVolumeUsageCommandProto());
processCommonCommand(commandResponseProto, refreshVolumeUsageCommand);
break;
+ case diskBalancerCommand:
+ DiskBalancerCommand diskBalancerCommand =
+ DiskBalancerCommand.getFromProtobuf(
+ commandResponseProto.getDiskBalancerCommandProto(), conf);
+ processCommonCommand(commandResponseProto, diskBalancerCommand);
+ break;
default:
throw new IllegalArgumentException("Unknown response : "
+ commandResponseProto.getCommandType().name());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index a1d159ef35..6bb202e3cc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
@@ -528,4 +529,9 @@ public class OzoneContainer {
public ContainerMetrics getMetrics() {
return metrics;
}
+
+ public DiskBalancerReportProto getDiskBalancerReport() {
+ // TODO: Return real disk balancer report
+ return null;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
new file mode 100644
index 0000000000..ea97f34ca0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerCommandProto;
+
+/**
+ * Informs a datanode to update DiskBalancer status.
+ */
+public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
+
+ private final boolean shouldRun;
+ private final DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public DiskBalancerCommand(final boolean shouldRun,
+ final DiskBalancerConfiguration diskBalancerConfiguration) {
+ this.shouldRun = shouldRun;
+ this.diskBalancerConfiguration = diskBalancerConfiguration;
+ }
+
+ /**
+ * Returns the type of this command.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ @Override
+ public DiskBalancerCommandProto getProto() {
+ return DiskBalancerCommandProto.newBuilder()
+ .setShouldRun(shouldRun)
+ .setDiskBalancerConf(diskBalancerConfiguration.toProtobufBuilder())
+ .build();
+ }
+
+ public static DiskBalancerCommand getFromProtobuf(DiskBalancerCommandProto
+ diskbalancerCommandProto, ConfigurationSource configuration) {
+ Preconditions.checkNotNull(diskbalancerCommandProto);
+ return new DiskBalancerCommand(diskbalancerCommandProto.getShouldRun(),
+ DiskBalancerConfiguration.fromProtobuf(
+ diskbalancerCommandProto.getDiskBalancerConf(), configuration));
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index d611caf126..8318f14b47 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -31,7 +31,9 @@ import org.apache.hadoop.hdds.HddsIdFactory;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
import org.apache.hadoop.hdds.protocol.proto.
@@ -39,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto.
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.junit.jupiter.api.Assertions;
@@ -215,6 +218,45 @@ public class TestReportPublisher {
executorService.shutdown();
}
+ @Test
+ public void testDiskBalancerReportPublisher() throws IOException {
+ StateContext dummyContext = Mockito.mock(StateContext.class);
+ DatanodeStateMachine dummyStateMachine =
+ Mockito.mock(DatanodeStateMachine.class);
+ OzoneContainer dummyContainer = Mockito.mock(OzoneContainer.class);
+
+ DiskBalancerReportProto.Builder builder =
+ DiskBalancerReportProto.newBuilder();
+ builder.setIsRunning(true);
+ builder.setBalancedBytes(1L);
+ builder.setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build());
+ DiskBalancerReportProto dummyReport = builder.build();
+
+ ReportPublisher publisher = new DiskBalancerReportPublisher();
+ when(dummyContext.getParent()).thenReturn(dummyStateMachine);
+ when(dummyStateMachine.getContainer()).thenReturn(dummyContainer);
+ when(dummyContainer.getDiskBalancerReport()).thenReturn(dummyReport);
+ publisher.setConf(config);
+
+ ScheduledExecutorService executorService = HadoopExecutors
+ .newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("Unit test ReportManager Thread - %d").build());
+ publisher.init(dummyContext, executorService);
+ Message report =
+ ((DiskBalancerReportPublisher) publisher).getReport();
+ Assertions.assertNotNull(report);
+ for (Descriptors.FieldDescriptor descriptor :
+ report.getDescriptorForType().getFields()) {
+ if (descriptor.getNumber() ==
+ DiskBalancerReportProto.ISRUNNING_FIELD_NUMBER) {
+ Assertions.assertEquals(true, report.getField(descriptor));
+ }
+ }
+ executorService.shutdown();
+ }
+
/**
* Get a datanode details.
*
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
index ea915f98a2..0b43bb93f9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.junit.Assert;
import org.junit.Rule;
@@ -66,6 +67,17 @@ public class TestReportPublisherFactory {
Assert.assertEquals(conf, publisher.getConf());
}
+ @Test
+ public void testGetDiskBalancerReportPublisher() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ ReportPublisherFactory factory = new ReportPublisherFactory(conf);
+ ReportPublisher publisher = factory
+ .getPublisherFor(DiskBalancerReportProto.class);
+ Assert.assertEquals(DiskBalancerReportPublisher.class,
+ publisher.getClass());
+ Assert.assertEquals(conf, publisher.getConf());
+ }
+
@Test
public void testInvalidReportPublisher() {
OzoneConfiguration conf = new OzoneConfiguration();
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index dfb9873adc..0e177449d1 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -49,6 +49,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1081,11 +1084,14 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
DatanodeDiskBalancerInfoRequestProto.newBuilder()
.setInfoType(DatanodeDiskBalancerInfoType.status);
hosts.ifPresent(requestBuilder::addAllHosts);
+ status.ifPresent(requestBuilder::setStatus);
DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
DatanodeDiskBalancerInfoResponseProto response =
@@ -1097,22 +1103,83 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.start)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.stop);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.update)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 26fdd47ca4..e6f5fbde79 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -80,7 +80,8 @@ message ScmContainerLocationRequest {
optional ResetDeletedBlockRetryCountRequestProto resetDeletedBlockRetryCountRequest = 41;
optional TransferLeadershipRequestProto transferScmLeadershipRequest = 42;
optional GetFailedDeletedBlocksTxnRequestProto getFailedDeletedBlocksTxnRequest = 43;
- optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 44;
+ optional DatanodeDiskBalancerInfoRequestProto datanodeDiskBalancerInfoRequest = 44;
+ optional DatanodeDiskBalancerOpRequestProto datanodeDiskBalancerOpRequest = 45;
}
message ScmContainerLocationResponse {
@@ -132,7 +133,8 @@ message ScmContainerLocationResponse {
optional ResetDeletedBlockRetryCountResponseProto resetDeletedBlockRetryCountResponse = 41;
optional TransferLeadershipResponseProto transferScmLeadershipResponse = 42;
optional GetFailedDeletedBlocksTxnResponseProto getFailedDeletedBlocksTxnResponse = 43;
- optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 44;
+ optional DatanodeDiskBalancerInfoResponseProto datanodeDiskBalancerInfoResponse = 44;
+ optional DatanodeDiskBalancerOpResponseProto datanodeDiskBalancerOpResponse = 45;
enum Status {
OK = 1;
@@ -184,6 +186,7 @@ enum Type {
TransferLeadership = 38;
GetFailedDeletedBlocksTransaction = 39;
DatanodeDiskBalancerInfo = 40;
+ DatanodeDiskBalancerOp = 41;
}
/**
@@ -347,12 +350,29 @@ message DatanodeDiskBalancerInfoRequestProto {
required DatanodeDiskBalancerInfoType infoType = 1;
optional uint32 count = 2;
repeated string hosts = 3;
+ optional DiskBalancerRunningStatus status = 4;
}
message DatanodeDiskBalancerInfoResponseProto {
repeated DatanodeDiskBalancerInfoProto info = 1;
}
+enum DatanodeDiskBalancerOpType{
+ start = 1;
+ stop = 2;
+ update = 3;
+}
+
+message DatanodeDiskBalancerOpRequestProto {
+ required DatanodeDiskBalancerOpType opType = 1;
+ repeated string hosts = 2;
+ optional DiskBalancerConfigurationProto conf = 3;
+}
+
+message DatanodeDiskBalancerOpResponseProto {
+ repeated DatanodeAdminErrorResponseProto failedHosts = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index f251449223..4cf60338d2 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -469,13 +469,19 @@ message DeletedBlocksTransactionInfo {
message DiskBalancerConfigurationProto {
optional double threshold = 1;
- optional double diskBandwidth = 2;
+ optional uint64 diskBandwidthInMB = 2;
optional int32 parallelThread = 3;
}
+enum DiskBalancerRunningStatus {
+ RUNNING = 1;
+ STOPPED = 2;
+ UNKNOWN = 3;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
- optional bool diskBalancerRunning = 3;
+ optional DiskBalancerRunningStatus runningStatus = 3;
optional DiskBalancerConfigurationProto diskBalancerConf = 4;
}
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 665665ee38..81e9ee54d3 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -139,6 +139,7 @@ message SCMHeartbeatRequestProto {
optional PipelineReportsProto pipelineReports = 8;
optional LayoutVersionProto dataNodeLayoutVersion = 9;
optional CommandQueueReportProto commandQueueReport = 10;
+ optional DiskBalancerReportProto diskBalancerReport = 11;
}
message CommandQueueReportProto {
@@ -325,6 +326,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommand = 9;
refreshVolumeUsageInfo = 10;
reconstructECContainersCommand = 11;
+ diskBalancerCommand = 12;
}
// TODO: once we start using protoc 3.x, refactor this message using "oneof"
required Type commandType = 1;
@@ -340,6 +342,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommandProto = 10;
optional RefreshVolumeUsageCommandProto refreshVolumeUsageCommandProto = 11;
optional ReconstructECContainersCommandProto reconstructECContainersCommandProto = 12;
+ optional DiskBalancerCommandProto diskBalancerCommandProto = 13;
// If running upon Ratis, holds term of underlying RaftServer iff current
@@ -442,6 +445,14 @@ message DatanodeDetailsAndReplicaIndexProto {
required int32 replicaIndex = 2;
}
+/**
+This command asks the datanode to update diskBalancer status
+ */
+message DiskBalancerCommandProto {
+ required bool shouldRun = 1;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 2;
+}
+
/**
This command asks the datanode to create a pipeline.
*/
@@ -480,6 +491,12 @@ message CRLStatusReport {
repeated int64 pendingCrlIds=2;
}
+message DiskBalancerReportProto {
+ required bool isRunning = 1;
+ optional uint64 balancedBytes = 2;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 3;
+}
+
/**
* This command asks the datanode to process a new CRL.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 0cc205b2ff..d22a878249 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatu
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
@@ -217,6 +218,14 @@ public final class SCMEvents {
new TypedEvent<>(CRLStatusReportFromDatanode.class,
"Crl_Status_Report");
+ /**
+ * DiskBalancer reports are send out by Datanodes. This report is received by
+ * SCMDatanodeHeartbeatDispatcher and DiskBalancer_Report Event is generated.
+ */
+ public static final TypedEvent<DiskBalancerReportFromDatanode>
+ DISK_BALANCER_REPORT = new TypedEvent<>(
+ DiskBalancerReportFromDatanode.class, "DiskBalancer_Report");
+
/**
* Private Ctor. Never Constructed.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index 5ca66e9eef..c6114c52f5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -22,7 +22,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -47,10 +48,13 @@ public class DiskBalancerManager {
public static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private OzoneConfiguration conf;
private final EventPublisher scmNodeEventPublisher;
private final SCMContext scmContext;
private final NodeManager nodeManager;
private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private Map<DatanodeDetails, Long> balancedBytesMap;
private boolean useHostnames;
/**
@@ -60,6 +64,7 @@ public class DiskBalancerManager {
EventPublisher eventPublisher,
SCMContext scmContext,
NodeManager nodeManager) {
+ this.conf = conf;
this.scmNodeEventPublisher = eventPublisher;
this.scmContext = scmContext;
this.nodeManager = nodeManager;
@@ -95,42 +100,60 @@ public class DiskBalancerManager {
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus = status.orElse(null);
+
+ if (filterDns != null) {
+ return filterDns.stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(nodeManager::getDatanodeInfo)
+ .map(dn -> getInfoProto(dn, clientVersion))
+ .collect(Collectors.toList());
+ } else {
+ return nodeManager.getAllNodes().stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(dn -> getInfoProto((DatanodeInfo)dn, clientVersion))
+ .collect(Collectors.toList());
}
- return statusList;
}
- private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ private boolean shouldReturnDatanode(
+ HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
- if (hosts == null || hosts.isEmpty()) {
- return isRunning(datanodeDetails);
- } else {
- return hosts.contains(datanodeDetails);
+ boolean shouldReturn = true;
+ // If status specified, do not return if status not match.
+ if (status != null && getRunningStatus(datanodeDetails) != status) {
+ shouldReturn = false;
}
+ return shouldReturn;
}
+ private HddsProtos.DatanodeDiskBalancerInfoProto getInfoProto(
+ DatanodeInfo dn, int clientVersion) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(dn);
+ HddsProtos.DiskBalancerRunningStatus runningStatus =
+ getRunningStatus(dn);
+ HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
+ HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(dn.toProto(clientVersion))
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setRunningStatus(getRunningStatus(dn));
+ if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ builder.setDiskBalancerConf(statusMap.get(dn)
+ .getDiskBalancerConfiguration().toProtobufBuilder());
+ }
+ return builder.build();
+ }
/**
* Get volume density for a specific DatanodeDetails node.
*
@@ -144,8 +167,7 @@ public class DiskBalancerManager {
DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
double totalCapacity = 0d, totalUsed = 0d;
- for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
- datanodeInfo.getStorageReports()) {
+ for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
totalCapacity += reportProto.getCapacity();
totalUsed += reportProto.getScmUsed();
}
@@ -162,10 +184,17 @@ public class DiskBalancerManager {
return volumeDensitySum;
}
- private boolean isRunning(DatanodeDetails datanodeDetails) {
- return statusMap
- .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
- .isRunning();
+ private HddsProtos.DiskBalancerRunningStatus getRunningStatus(
+ DatanodeDetails datanodeDetails) {
+ if (!statusMap.containsKey(datanodeDetails)) {
+ return HddsProtos.DiskBalancerRunningStatus.UNKNOWN;
+ } else {
+ if (statusMap.get(datanodeDetails).isRunning()) {
+ return HddsProtos.DiskBalancerRunningStatus.RUNNING;
+ } else {
+ return HddsProtos.DiskBalancerRunningStatus.STOPPED;
+ }
+ }
}
@VisibleForTesting
@@ -173,4 +202,24 @@ public class DiskBalancerManager {
statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
new DiskBalancerConfiguration()));
}
-}
\ No newline at end of file
+
+ public void processDiskBalancerReport(DiskBalancerReportProto reportProto,
+ DatanodeDetails dn) {
+ boolean isRunning = reportProto.getIsRunning();
+ DiskBalancerConfiguration diskBalancerConfiguration =
+ reportProto.hasDiskBalancerConf() ?
+ DiskBalancerConfiguration.fromProtobuf(
+ reportProto.getDiskBalancerConf(), conf) :
+ new DiskBalancerConfiguration();
+ statusMap.put(dn, new DiskBalancerStatus(isRunning,
+ diskBalancerConfiguration));
+ if (reportProto.hasBalancedBytes()) {
+ balancedBytesMap.put(dn, reportProto.getBalancedBytes());
+ }
+ }
+
+ @VisibleForTesting
+ public Map<DatanodeDetails, DiskBalancerStatus> getStatusMap() {
+ return statusMap;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
new file mode 100644
index 0000000000..47158972ec
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles DiskBalancer Reports from datanode.
+ */
+public class DiskBalancerReportHandler implements
+ EventHandler<DiskBalancerReportFromDatanode> {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(DiskBalancerReportHandler.class);
+
+ private DiskBalancerManager diskBalancerManager;
+
+ public DiskBalancerReportHandler(DiskBalancerManager diskBalancerManager) {
+ this.diskBalancerManager = diskBalancerManager;
+ }
+
+ @Override
+ public void onMessage(DiskBalancerReportFromDatanode reportFromDatanode,
+ EventPublisher publisher) {
+ Preconditions.checkNotNull(reportFromDatanode);
+ DatanodeDetails dn = reportFromDatanode.getDatanodeDetails();
+ DiskBalancerReportProto diskBalancerReportProto =
+ reportFromDatanode.getReport();
+ Preconditions.checkNotNull(dn,
+ "DiskBalancer Report is missing DatanodeDetails.");
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("Processing diskBalancer report for dn: {}", dn);
+ }
+ try {
+ diskBalancerManager.processDiskBalancerReport(
+ diskBalancerReportProto, dn);
+ } catch (Exception e) {
+ LOGGER.error("Failed to process diskBalancer report={} from dn={}.",
+ diskBalancerReportProto, dn, e);
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
index ed22e80e3c..65f577bcee 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -32,8 +32,6 @@ public class DiskBalancerStatus {
private boolean isRunning;
private DiskBalancerConfiguration diskBalancerConfiguration;
- public static final DiskBalancerStatus DUMMY_STATUS =
- new DiskBalancerStatus(false, new DiskBalancerConfiguration());
public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
this.isRunning = isRunning;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 8d228b54e5..c1e41c0772 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -161,6 +161,14 @@ public interface NodeManager extends StorageContainerNodeProtocol,
*/
DatanodeUsageInfo getUsageInfo(DatanodeDetails dn);
+ /**
+ * Get the datanode info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeInfo of the specified datanode
+ */
+ DatanodeInfo getDatanodeInfo(DatanodeDetails dn);
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails DatanodeDetails.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 979ccc4b66..ff11d7a8f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -921,6 +921,23 @@ public class SCMNodeManager implements NodeManager {
return usageInfo;
}
+ /**
+ * Get the usage info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeUsageInfo of the specified datanode
+ */
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ try {
+ return nodeStateManager.getNode(dn);
+ } catch (NodeNotFoundException e) {
+ LOG.warn("Cannot retrieve DatanodeInfo, datanode {} not found.",
+ dn.getUuid());
+ return null;
+ }
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index b97d87989f..c656cd9ffb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -692,6 +693,13 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
request.getDatanodeDiskBalancerInfoRequest(),
request.getVersion()))
.build();
+ case DatanodeDiskBalancerOp:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerOpResponse(getDatanodeDiskBalancerOp(
+ request.getDatanodeDiskBalancerOpRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1232,7 +1240,9 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
break;
case status:
infoProtoList = impl.getDiskBalancerStatus(
- Optional.of(request.getHostsList()), clientVersion);
+ Optional.of(request.getHostsList()),
+ Optional.of(request.getStatus()),
+ clientVersion);
break;
default:
infoProtoList = null;
@@ -1241,4 +1251,43 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
.addAllInfo(infoProtoList)
.build();
}
+
+ public DatanodeDiskBalancerOpResponseProto getDatanodeDiskBalancerOp(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerOpRequestProto request)
+ throws IOException {
+ List<DatanodeAdminError> errors;
+ switch (request.getOpType()) {
+ case start:
+ errors = impl.startDiskBalancer(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case update:
+ errors = impl.updateDiskBalancerConfiguration(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case stop:
+ errors = impl.stopDiskBalancer(Optional.of(request.getHostsList()));
+ break;
+ default:
+ errors = new ArrayList<>();
+ }
+
+ DatanodeDiskBalancerOpResponseProto.Builder response =
+ DatanodeDiskBalancerOpResponseProto.newBuilder();
+ for (DatanodeAdminError e : errors) {
+ DatanodeAdminErrorResponseProto.Builder error =
+ DatanodeAdminErrorResponseProto.newBuilder();
+ error.setHost(e.getHostname());
+ error.setError(e.getError());
+ response.addFailedHosts(error);
+ }
+ return response.build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 7d85131d85..faa24a8021 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1228,7 +1228,9 @@ public class SCMClientProtocolServer implements
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
// check admin authorisation
try {
getScm().checkAdminAccess(getRemoteUser());
@@ -1237,29 +1239,33 @@ public class SCMClientProtocolServer implements
throw e;
}
- return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts, status,
clientVersion);
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index 6d9f42c49b..2fb360151b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
@@ -57,6 +58,7 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DISK_BALANCER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents
.INCREMENTAL_CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NODE_REPORT;
@@ -202,6 +204,15 @@ public final class SCMDatanodeHeartbeatDispatcher {
commandStatusReport));
}
}
+
+ if (heartbeat.hasDiskBalancerReport()) {
+ LOG.debug("Dispatching DiskBalancer Report.");
+ eventPublisher.fireEvent(
+ DISK_BALANCER_REPORT,
+ new DiskBalancerReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getDiskBalancerReport()));
+ }
}
return commands;
@@ -435,4 +446,16 @@ public final class SCMDatanodeHeartbeatDispatcher {
super(datanodeDetails, report);
}
}
+
+ /**
+ * DiskBalancer report event payload with origin.
+ */
+ public static class DiskBalancerReportFromDatanode
+ extends ReportFromDatanode<DiskBalancerReportProto> {
+
+ public DiskBalancerReportFromDatanode(DatanodeDetails datanodeDetails,
+ DiskBalancerReportProto report) {
+ super(datanodeDetails, report);
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 665a57a37f..b61ef218a0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerReportHandler;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
@@ -458,6 +459,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PipelineActionHandler(pipelineManager, scmContext, configuration);
CRLStatusReportHandler crlStatusReportHandler =
new CRLStatusReportHandler(certificateStore, configuration);
+ DiskBalancerReportHandler diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
@@ -530,7 +533,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
eventQueue.addHandler(SCMEvents.CRL_STATUS_REPORT, crlStatusReportHandler);
-
+ eventQueue.addHandler(SCMEvents.DISK_BALANCER_REPORT,
+ diskBalancerReportHandler);
}
private void initializeCertificateClient() {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index c10575925e..d6e9a7b8d2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -346,7 +346,36 @@ public class MockNodeManager implements NodeManager {
*/
@Override
public List<DatanodeDetails> getAllNodes() {
- return new ArrayList<>(nodeMetricMap.keySet());
+ // mock storage reports for TestDiskBalancer
+ List<DatanodeDetails> healthyNodesWithInfo = new ArrayList<>();
+ for (Map.Entry<DatanodeDetails, SCMNodeStat> entry:
+ nodeMetricMap.entrySet()) {
+ NodeStatus nodeStatus = NodeStatus.inServiceHealthy();
+ if (staleNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceStale();
+ } else if (deadNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceDead();
+ }
+ DatanodeInfo di = new DatanodeInfo(entry.getKey(), nodeStatus,
+ UpgradeUtils.defaultLayoutVersionProto());
+
+ long capacity = entry.getValue().getCapacity().get();
+ long used = entry.getValue().getScmUsed().get();
+ long remaining = entry.getValue().getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+
+ healthyNodesWithInfo.add(di);
+ }
+ return healthyNodesWithInfo;
}
/**
@@ -409,6 +438,26 @@ public class MockNodeManager implements NodeManager {
return new DatanodeUsageInfo(datanodeDetails, stat);
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dd) {
+ DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(),
+ UpgradeUtils.defaultLayoutVersionProto());
+ long capacity = nodeMetricMap.get(dd).getCapacity().get();
+ long used = nodeMetricMap.get(dd).getScmUsed().get();
+ long remaining = nodeMetricMap.get(dd).getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+ return di;
+ }
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails - datanode details.
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
index 69fb5ffa25..d35f8a7d71 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
@@ -244,6 +244,11 @@ public class SimpleMockNodeManager implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
return null;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
index 541c9764b7..7005dea292 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -21,8 +21,10 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.ozone.test.GenericTestUtils;
@@ -33,6 +35,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
+import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class TestDiskBalancerManager {
private NodeManager nodeManager;
private OzoneConfiguration conf;
private String storageDir;
+ private DiskBalancerReportHandler diskBalancerReportHandler;
+ private Random random;
@BeforeEach
public void setup() throws Exception {
@@ -56,6 +61,9 @@ public class TestDiskBalancerManager {
nodeManager = new MockNodeManager(true, 3);
diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
SCMContext.emptyContext(), nodeManager);
+ diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
+ random = new Random();
}
@Test
@@ -73,6 +81,7 @@ public class TestDiskBalancerManager {
@Test
public void testDatanodeDiskBalancerStatus() throws IOException {
diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(1));
// Simulate users asking all status of 3 datanodes
List<String> dns = nodeManager.getAllNodes().stream().map(
@@ -81,6 +90,7 @@ public class TestDiskBalancerManager {
List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(3, statusProtoList.size());
@@ -92,8 +102,32 @@ public class TestDiskBalancerManager {
statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(1, statusProtoList.size());
}
+
+ @Test
+ public void testHandleDiskBalancerReportFromDatanode() {
+ for (DatanodeDetails dn: nodeManager.getAllNodes()) {
+ diskBalancerReportHandler.onMessage(
+ new DiskBalancerReportFromDatanode(dn, generateRandomReport()), null);
+ }
+
+ Assertions.assertEquals(3, diskBalancerManager.getStatusMap().size());
+ }
+
+ private DiskBalancerReportProto generateRandomReport() {
+ return DiskBalancerReportProto.newBuilder()
+ .setIsRunning(random.nextBoolean())
+ .setBalancedBytes(random.nextInt(10000))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(random.nextInt(99))
+ .setParallelThread(random.nextInt(4) + 1)
+ .setDiskBandwidthInMB(random.nextInt(99) + 1)
+ .build())
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index f3e082cf43..b9e29aa009 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -194,6 +195,11 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 1c0535128d..29502b953e 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -518,31 +518,34 @@ public class ContainerOperationClient implements ScmClient {
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
- storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
- hosts);
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.startDiskBalancer(threshold,
+ bandwidthInMB, parallelThread, hosts);
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.stopDiskBalancer(hosts);
+ return storageContainerLocationClient.stopDiskBalancer(hosts);
}
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException {
return storageContainerLocationClient.getDiskBalancerStatus(hosts,
- ClientVersion.CURRENT_VERSION);
+ runningStatus, ClientVersion.CURRENT_VERSION);
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidth,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
- bandwidth, hosts);
+ return storageContainerLocationClient.updateDiskBalancerConfiguration(
+ threshold, bandwidth, parallelThread, hosts);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
index 267209cf03..32719f0d83 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -54,6 +55,7 @@ public class TestDiskBalancer {
private static ScmClient storageClient;
private static MiniOzoneCluster cluster;
private static OzoneConfiguration ozoneConf;
+ private static DiskBalancerManager diskBalancerManager;
@BeforeClass
public static void setup() throws Exception {
@@ -63,6 +65,8 @@ public class TestDiskBalancer {
cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
storageClient = new ContainerOperationClient(ozoneConf);
cluster.waitForClusterToBeReady();
+ diskBalancerManager = cluster.getStorageContainerManager()
+ .getDiskBalancerManager();
for (DatanodeDetails dn: cluster.getStorageContainerManager()
.getScmNodeManager().getAllNodes()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/05: HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 9094b0e839820d7ba8a1d672193596326719c9d9
Author: Symious <yi...@foxmail.com>
AuthorDate: Tue Aug 16 10:40:22 2022 +0800
HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 41 +++++
.../protocol/StorageContainerLocationProtocol.java | 30 ++++
.../scm/storage/DiskBalancerConfiguration.java | 165 +++++++++++++++++++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 55 +++++++
.../src/main/proto/ScmAdminProtocol.proto | 21 +++
.../interface-client/src/main/proto/hdds.proto | 15 +-
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 176 +++++++++++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 50 ++++++
.../hdds/scm/node/NodeDecommissionManager.java | 123 +-------------
.../org/apache/hadoop/hdds/scm/node/NodeUtils.java | 150 ++++++++++++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 31 ++++
.../hdds/scm/server/SCMClientProtocolServer.java | 50 ++++++
.../hdds/scm/server/StorageContainerManager.java | 8 +
.../hdds/scm/node/TestDiskBalancerManager.java | 99 ++++++++++++
.../hdds/scm/node/TestNodeDecommissionManager.java | 13 +-
.../hdds/scm/cli/ContainerOperationClient.java | 36 +++++
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 96 +++++++++++
18 files changed, 1036 insertions(+), 126 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 916bd7fdfb..d4f6267c49 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -420,4 +420,45 @@ public interface ScmClient extends Closeable {
StatusAndMessages queryUpgradeFinalizationProgress(
String upgradeClientID, boolean force, boolean readonly)
throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param count top datanodes that need balancing
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param hosts If hosts is not null, return status of hosts; If hosts is
+ * null, return status of all datanodes in balancing.
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 7690b2eefb..d3e02a5104 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -455,4 +455,34 @@ public interface StorageContainerLocationProtocol extends Closeable {
List<ContainerInfo> getListOfContainers(
long startContainerID, int count, HddsProtos.LifeCycleState state)
throws IOException;
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
new file mode 100644
index 0000000000..704b383679
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class contains configuration values for the DiskBalancer.
+ */
+@ConfigGroup(prefix = "hdds.datanode.disk.balancer")
+public final class DiskBalancerConfiguration {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerConfiguration.class);
+
+ @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "Threshold is a percentage in the range of 0 to 100. A " +
+ "datanode is considered balanced if for each volume, the " +
+ "utilization of the volume(used space to capacity ratio) differs" +
+ " from the utilization of the datanode(used space to capacity ratio" +
+ " of the entire datanode) no more than the threshold.")
+ private double threshold = 10d;
+
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "The max balance speed.")
+ private double diskBandwidth = 10;
+
+ @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
+ description = "The max parallel balance thread count.")
+ private int parallelThread = 5;
+
+ /**
+ * Gets the threshold value for DiskBalancer.
+ *
+ * @return percentage value in the range 0 to 100
+ */
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public double getThresholdAsRatio() {
+ return threshold / 100;
+ }
+
+ /**
+ * Sets the threshold value for Disk Balancer.
+ *
+ * @param threshold a percentage value in the range 0 to 100
+ */
+ public void setThreshold(double threshold) {
+ if (threshold < 0d || threshold >= 100d) {
+ throw new IllegalArgumentException(
+ "Threshold must be a percentage(double) in the range 0 to 100.");
+ }
+ this.threshold = threshold;
+ }
+
+ /**
+ * Gets the disk bandwidth value for Disk Balancer.
+ *
+ * @return max disk bandwidth per second
+ */
+ public double getDiskBandwidth() {
+ return diskBandwidth;
+ }
+
+ /**
+ * Sets the disk bandwidth value for Disk Balancer.
+ *
+ * @param diskBandwidth the bandwidth to control balance speed
+ */
+ public void setDiskBandwidth(double diskBandwidth) {
+ if (diskBandwidth <= 0d) {
+ throw new IllegalArgumentException(
+ "diskBandwidth must be a value larger than 0.");
+ }
+ this.diskBandwidth = diskBandwidth;
+ }
+
+ /**
+ * Gets the parallel thread for Disk Balancer.
+ *
+ * @return parallel thread
+ */
+ public int getParallelThread() {
+ return parallelThread;
+ }
+
+ /**
+ * Sets the parallel thread for Disk Balancer.
+ *
+ * @param parallelThread the parallel thread count
+ */
+ public void setParallelThread(int parallelThread) {
+ if (parallelThread <= 0) {
+ throw new IllegalArgumentException(
+ "parallelThread must be a value larger than 0.");
+ }
+ this.parallelThread = parallelThread;
+ }
+ @Override
+ public String toString() {
+ return String.format("Disk Balancer Configuration values:%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n",
+ "Key", "Value",
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Parallel Thread", parallelThread);
+ }
+
+ public HddsProtos.DiskBalancerConfigurationProto.Builder toProtobufBuilder() {
+ HddsProtos.DiskBalancerConfigurationProto.Builder builder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+
+ builder.setThreshold(threshold)
+ .setDiskBandwidth(diskBandwidth)
+ .setParallelThread(parallelThread);
+ return builder;
+ }
+
+ static DiskBalancerConfiguration fromProtobuf(
+ @NotNull HddsProtos.DiskBalancerConfigurationProto proto,
+ @NotNull OzoneConfiguration ozoneConfiguration) {
+ DiskBalancerConfiguration config =
+ ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ if (proto.hasThreshold()) {
+ config.setThreshold(proto.getThreshold());
+ }
+ if (proto.hasDiskBandwidth()) {
+ config.setDiskBandwidth(proto.getDiskBandwidth());
+ }
+ if (proto.hasParallelThread()) {
+ config.setParallelThread(proto.getParallelThread());
+ }
+ return config;
+ }
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 24feb69389..d4817ea592 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@ -52,5 +52,6 @@ public enum ConfigTag {
TLS,
TOKEN,
UPGRADE,
- X509
+ X509,
+ DISKBALANCER
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index c664a42ae6..dfb9873adc 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -46,6 +46,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1060,6 +1063,58 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
return response.getContainerCount();
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto request =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.report)
+ .setCount(count)
+ .build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.status);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ }
+
@Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index d5a3c6f65a..26fdd47ca4 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -80,6 +80,7 @@ message ScmContainerLocationRequest {
optional ResetDeletedBlockRetryCountRequestProto resetDeletedBlockRetryCountRequest = 41;
optional TransferLeadershipRequestProto transferScmLeadershipRequest = 42;
optional GetFailedDeletedBlocksTxnRequestProto getFailedDeletedBlocksTxnRequest = 43;
+ optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 44;
}
message ScmContainerLocationResponse {
@@ -131,6 +132,7 @@ message ScmContainerLocationResponse {
optional ResetDeletedBlockRetryCountResponseProto resetDeletedBlockRetryCountResponse = 41;
optional TransferLeadershipResponseProto transferScmLeadershipResponse = 42;
optional GetFailedDeletedBlocksTxnResponseProto getFailedDeletedBlocksTxnResponse = 43;
+ optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 44;
enum Status {
OK = 1;
@@ -181,6 +183,7 @@ enum Type {
GetClosedContainerCount = 37;
TransferLeadership = 38;
GetFailedDeletedBlocksTransaction = 39;
+ DatanodeDiskBalancerInfo = 40;
}
/**
@@ -332,6 +335,24 @@ message DatanodeUsageInfoResponseProto {
repeated DatanodeUsageInfoProto info = 1;
}
+enum DatanodeDiskBalancerInfoType{
+ report = 1;
+ status = 2;
+}
+
+/*
+ Datanode disk balancer status request message.
+*/
+message DatanodeDiskBalancerInfoRequestProto {
+ required DatanodeDiskBalancerInfoType infoType = 1;
+ optional uint32 count = 2;
+ repeated string hosts = 3;
+}
+
+message DatanodeDiskBalancerInfoResponseProto {
+ repeated DatanodeDiskBalancerInfoProto info = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 16ea4887aa..f251449223 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -465,4 +465,17 @@ message DeletedBlocksTransactionInfo {
optional int64 containerID = 2;
repeated int64 localID = 3;
optional int32 count = 4;
-}
\ No newline at end of file
+}
+
+message DiskBalancerConfigurationProto {
+ optional double threshold = 1;
+ optional double diskBandwidth = 2;
+ optional int32 parallelThread = 3;
+}
+
+message DatanodeDiskBalancerInfoProto {
+ required DatanodeDetailsProto node = 1;
+ required double currentVolumeDensitySum = 2;
+ optional bool diskBalancerRunning = 3;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 4;
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
new file mode 100644
index 0000000000..5ca66e9eef
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+
+/**
+ * Maintains information about the DiskBalancer on SCM side.
+ */
+public class DiskBalancerManager {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private final EventPublisher scmNodeEventPublisher;
+ private final SCMContext scmContext;
+ private final NodeManager nodeManager;
+ private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private boolean useHostnames;
+
+ /**
+ * Constructs DiskBalancer Manager.
+ */
+ public DiskBalancerManager(OzoneConfiguration conf,
+ EventPublisher eventPublisher,
+ SCMContext scmContext,
+ NodeManager nodeManager) {
+ this.scmNodeEventPublisher = eventPublisher;
+ this.scmContext = scmContext;
+ this.nodeManager = nodeManager;
+ this.useHostnames = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
+ this.statusMap = new ConcurrentHashMap<>();
+ }
+
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportList =
+ new ArrayList<>();
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ reportList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+
+ reportList.sort((t1, t2) -> Double.compare(t2.getCurrentVolumeDensitySum(),
+ t1.getCurrentVolumeDensitySum()));
+ return reportList.stream().limit(count).collect(Collectors.toList());
+ }
+
+ /**
+ * If hosts is not null, return status of hosts;
+ * If hosts is null, return status of all datanodes in balancing.
+ */
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
+ new ArrayList<>();
+ List<DatanodeDetails> filterDns = null;
+ if (hosts.isPresent() && !hosts.get().isEmpty()) {
+ filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ }
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ if (shouldReturnDatanode(filterDns, datanodeDetails)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setDiskBalancerRunning(isRunning(datanodeDetails))
+ .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
+ DiskBalancerStatus.DUMMY_STATUS)
+ .getDiskBalancerConfiguration().toProtobufBuilder())
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+ }
+ return statusList;
+ }
+
+ private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ DatanodeDetails datanodeDetails) {
+ if (hosts == null || hosts.isEmpty()) {
+ return isRunning(datanodeDetails);
+ } else {
+ return hosts.contains(datanodeDetails);
+ }
+ }
+
+ /**
+ * Get volume density for a specific DatanodeDetails node.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @return DiskBalancer report.
+ */
+ private double getVolumeDataDensitySumForDatanodeDetails(
+ DatanodeDetails datanodeDetails) {
+ Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo);
+
+ DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
+
+ double totalCapacity = 0d, totalUsed = 0d;
+ for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
+ datanodeInfo.getStorageReports()) {
+ totalCapacity += reportProto.getCapacity();
+ totalUsed += reportProto.getScmUsed();
+ }
+
+ Preconditions.checkArgument(totalCapacity != 0);
+ double idealUsage = totalUsed / totalCapacity;
+
+ double volumeDensitySum = datanodeInfo.getStorageReports().stream()
+ .map(report ->
+ Math.abs((double)report.getScmUsed() / report.getCapacity()
+ - idealUsage))
+ .mapToDouble(Double::valueOf).sum();
+
+ return volumeDensitySum;
+ }
+
+ private boolean isRunning(DatanodeDetails datanodeDetails) {
+ return statusMap
+ .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
+ .isRunning();
+ }
+
+ @VisibleForTesting
+ public void addRunningDatanode(DatanodeDetails datanodeDetails) {
+ statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
+ new DiskBalancerConfiguration()));
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
new file mode 100644
index 0000000000..ed22e80e3c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Maintains DiskBalancerConfiguration and isRunning.
+ */
+public class DiskBalancerStatus {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerStatus.class);
+
+ private boolean isRunning;
+ private DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public static final DiskBalancerStatus DUMMY_STATUS =
+ new DiskBalancerStatus(false, new DiskBalancerConfiguration());
+
+ public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
+ this.isRunning = isRunning;
+ this.diskBalancerConfiguration = conf;
+ }
+
+ public boolean isRunning() {
+ return isRunning;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
index a84b07d513..80f81782ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
@@ -32,12 +32,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.util.ArrayList;
-import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -66,115 +61,6 @@ public class NodeDecommissionManager {
private static final Logger LOG =
LoggerFactory.getLogger(NodeDecommissionManager.class);
- static class HostDefinition {
- private String rawHostname;
- private String hostname;
- private int port;
-
- HostDefinition(String hostname) throws InvalidHostStringException {
- this.rawHostname = hostname;
- parseHostname();
- }
-
- public String getRawHostname() {
- return rawHostname;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public int getPort() {
- return port;
- }
-
- private void parseHostname() throws InvalidHostStringException {
- try {
- // A URI *must* have a scheme, so just create a fake one
- URI uri = new URI("empty://" + rawHostname.trim());
- this.hostname = uri.getHost();
- this.port = uri.getPort();
-
- if (this.hostname == null) {
- throw new InvalidHostStringException("The string " + rawHostname +
- " does not contain a value hostname or hostname:port definition");
- }
- } catch (URISyntaxException e) {
- throw new InvalidHostStringException(
- "Unable to parse the hoststring " + rawHostname, e);
- }
- }
- }
-
- private List<DatanodeDetails> mapHostnamesToDatanodes(List<String> hosts)
- throws InvalidHostStringException {
- List<DatanodeDetails> results = new LinkedList<>();
- for (String hostString : hosts) {
- HostDefinition host = new HostDefinition(hostString);
- InetAddress addr;
- try {
- addr = InetAddress.getByName(host.getHostname());
- } catch (UnknownHostException e) {
- throw new InvalidHostStringException("Unable to resolve host "
- + host.getRawHostname(), e);
- }
- String dnsName;
- if (useHostnames) {
- dnsName = addr.getHostName();
- } else {
- dnsName = addr.getHostAddress();
- }
- List<DatanodeDetails> found = nodeManager.getNodesByAddress(dnsName);
- if (found.size() == 0) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " (" + dnsName + ") is not running any datanodes registered"
- + " with SCM."
- + " Please check the host name.");
- } else if (found.size() == 1) {
- if (host.getPort() != -1 &&
- !validateDNPortMatch(host.getPort(), found.get(0))) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " is running a datanode registered with SCM,"
- + " but the port number doesn't match."
- + " Please check the port number.");
- }
- results.add(found.get(0));
- } else if (found.size() > 1) {
- DatanodeDetails match = null;
- for (DatanodeDetails dn : found) {
- if (validateDNPortMatch(host.getPort(), dn)) {
- match = dn;
- break;
- }
- }
- if (match == null) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " is running multiple datanodes registered with SCM,"
- + " but no port numbers match."
- + " Please check the port number.");
- }
- results.add(match);
- }
- }
- return results;
- }
-
- /**
- * Check if the passed port is used by the given DatanodeDetails object. If
- * it is, return true, otherwise return false.
- * @param port Port number to check if it is used by the datanode
- * @param dn Datanode to check if it is using the given port
- * @return True if port is used by the datanode. False otherwise.
- */
- private boolean validateDNPortMatch(int port, DatanodeDetails dn) {
- for (DatanodeDetails.Port p : dn.getPorts()) {
- if (p.getValue() == port) {
- return true;
- }
- }
- return false;
- }
-
public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm,
ContainerManager containerManager, SCMContext scmContext,
EventPublisher eventQueue, ReplicationManager rm) {
@@ -225,7 +111,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> decommissionNodes(
List<String> nodes) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
@@ -290,7 +177,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> recommissionNodes(
List<String> nodes) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
@@ -327,7 +215,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> startMaintenanceNodes(
List<String> nodes, int endInHours) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
new file mode 100644
index 0000000000..6df74d3746
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Util class for Node operations.
+ */
+public final class NodeUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NodeUtils.class);
+
+ private NodeUtils() {
+ }
+
+ public static List<DatanodeDetails> mapHostnamesToDatanodes(
+ NodeManager nodeManager, List<String> hosts, boolean useHostnames)
+ throws InvalidHostStringException {
+ List<DatanodeDetails> results = new LinkedList<>();
+ for (String hostString : hosts) {
+ HostDefinition host = new HostDefinition(hostString);
+ InetAddress addr;
+ try {
+ addr = InetAddress.getByName(host.getHostname());
+ } catch (UnknownHostException e) {
+ throw new InvalidHostStringException("Unable to resolve host "
+ + host.getRawHostname(), e);
+ }
+ String dnsName;
+ if (useHostnames) {
+ dnsName = addr.getHostName();
+ } else {
+ dnsName = addr.getHostAddress();
+ }
+ List<DatanodeDetails> found = nodeManager.getNodesByAddress(dnsName);
+ if (found.size() == 0) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " (" + dnsName + ") is not running any datanodes registered"
+ + " with SCM."
+ + " Please check the host name.");
+ } else if (found.size() == 1) {
+ if (host.getPort() != -1 &&
+ !validateDNPortMatch(host.getPort(), found.get(0))) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running a datanode registered with SCM,"
+ + " but the port number doesn't match."
+ + " Please check the port number.");
+ }
+ results.add(found.get(0));
+ } else if (found.size() > 1) {
+ DatanodeDetails match = null;
+ for (DatanodeDetails dn : found) {
+ if (validateDNPortMatch(host.getPort(), dn)) {
+ match = dn;
+ break;
+ }
+ }
+ if (match == null) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running multiple datanodes registered with SCM,"
+ + " but no port numbers match."
+ + " Please check the port number.");
+ }
+ results.add(match);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Check if the passed port is used by the given DatanodeDetails object. If
+ * it is, return true, otherwise return false.
+ * @param port Port number to check if it is used by the datanode
+ * @param dn Datanode to check if it is using the given port
+ * @return True if port is used by the datanode. False otherwise.
+ */
+ private static boolean validateDNPortMatch(int port, DatanodeDetails dn) {
+ for (DatanodeDetails.Port p : dn.getPorts()) {
+ if (p.getValue() == port) {
+ return true;
+ }
+ }
+ return false;
+ }
+ static class HostDefinition {
+ private String rawHostname;
+ private String hostname;
+ private int port;
+
+ HostDefinition(String hostname) throws InvalidHostStringException {
+ this.rawHostname = hostname;
+ parseHostname();
+ }
+
+ public String getRawHostname() {
+ return rawHostname;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ private void parseHostname() throws InvalidHostStringException {
+ try {
+ // A URI *must* have a scheme, so just create a fake one
+ URI uri = new URI("empty://" + rawHostname.trim());
+ this.hostname = uri.getHost();
+ this.port = uri.getPort();
+
+ if (this.hostname == null) {
+ throw new InvalidHostStringException("The string " + rawHostname +
+ " does not contain a value hostname or hostname:port definition");
+ }
+ } catch (URISyntaxException e) {
+ throw new InvalidHostStringException(
+ "Unable to parse the hoststring " + rawHostname, e);
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 2dc3c5b04f..b97d87989f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -683,6 +684,14 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
transferScmLeadership(
request.getTransferScmLeadershipRequest()))
.build();
+ case DatanodeDiskBalancerInfo:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerInfoResponse(getDatanodeDiskBalancerInfo(
+ request.getDatanodeDiskBalancerInfoRequest(),
+ request.getVersion()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1210,4 +1219,26 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
impl.transferLeadership(newLeaderId);
return TransferLeadershipResponseProto.getDefaultInstance();
}
+
+ public DatanodeDiskBalancerInfoResponseProto getDatanodeDiskBalancerInfo(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerInfoRequestProto request, int clientVersion)
+ throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> infoProtoList;
+ switch (request.getInfoType()) {
+ case report:
+ infoProtoList = impl.getDiskBalancerReport(request.getCount(),
+ clientVersion);
+ break;
+ case status:
+ infoProtoList = impl.getDiskBalancerStatus(
+ Optional.of(request.getHostsList()), clientVersion);
+ break;
+ default:
+ infoProtoList = null;
+ }
+ return DatanodeDiskBalancerInfoResponseProto.newBuilder()
+ .addAllInfo(infoProtoList)
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index ccd4153698..7d85131d85 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1212,6 +1212,56 @@ public class SCMClientProtocolServer implements
ContainerID.valueOf(startContainerID), count, state);
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerReport(count,
+ clientVersion);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ clientVersion);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
/**
* Queries a list of Node that match a set of statuses.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 035958a39c..665a57a37f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
@@ -234,6 +235,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private WritableContainerFactory writableContainerFactory;
private FinalizationManager finalizationManager;
private HDDSLayoutVersionManager scmLayoutVersionManager;
+ private DiskBalancerManager diskBalancerManager;
private SCMMetadataStore scmMetadataStore;
private CertificateStore certificateStore;
@@ -769,6 +771,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
.setSCMDBTransactionBuffer(scmHAManager.getDBTransactionBuffer())
.setRatisServer(scmHAManager.getRatisServer())
.build();
+ diskBalancerManager = new DiskBalancerManager(conf, eventQueue, scmContext,
+ scmNodeManager);
}
/**
@@ -1840,6 +1844,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return serviceManager;
}
+ public DiskBalancerManager getDiskBalancerManager() {
+ return diskBalancerManager;
+ }
+
/**
* Force SCM out of safe mode.
*/
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
new file mode 100644
index 0000000000..541c9764b7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Unit tests for the DiskBalancer manager.
+ */
+
+public class TestDiskBalancerManager {
+
+ private DiskBalancerManager diskBalancerManager;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private String storageDir;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ storageDir = GenericTestUtils.getTempPath(
+ TestDiskBalancerManager.class.getSimpleName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ nodeManager = new MockNodeManager(true, 3);
+ diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
+ SCMContext.emptyContext(), nodeManager);
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ diskBalancerManager.getDiskBalancerReport(2,
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(2, reportProtoList.size());
+ Assertions.assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+
+ // Simulate users asking all status of 3 datanodes
+ List<String> dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).collect(
+ Collectors.toList());
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(3, statusProtoList.size());
+
+ // Simulate users asking status of 1 datanodes
+ dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).limit(1).collect(
+ Collectors.toList());
+
+ statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(1, statusProtoList.size());
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
index 7922aa13de..e60db61165 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeUtils.HostDefinition;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -67,26 +68,24 @@ public class TestNodeDecommissionManager {
@Test
public void testHostStringsParseCorrectly()
throws InvalidHostStringException {
- NodeDecommissionManager.HostDefinition def =
- new NodeDecommissionManager.HostDefinition("foobar");
+ HostDefinition def = new HostDefinition("foobar");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(" foobar ");
+ def = new HostDefinition(" foobar ");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition("foobar:1234");
+ def = new HostDefinition("foobar:1234");
assertEquals("foobar", def.getHostname());
assertEquals(1234, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(
- "foobar.mycompany.com:1234");
+ def = new HostDefinition("foobar.mycompany.com:1234");
assertEquals("foobar.mycompany.com", def.getHostname());
assertEquals(1234, def.getPort());
try {
- new NodeDecommissionManager.HostDefinition("foobar:abcd");
+ new HostDefinition("foobar:abcd");
fail("InvalidHostStringException should have been thrown");
} catch (InvalidHostStringException e) {
}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 22050f1940..1c0535128d 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -509,4 +509,40 @@ public class ContainerOperationClient implements ScmClient {
return storageContainerLocationClient.queryUpgradeFinalizationProgress(
upgradeClientID, force, readonly);
}
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerReport(count,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
+ hosts);
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.stopDiskBalancer(hosts);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerStatus(hosts,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
+ bandwidth, hosts);
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
new file mode 100644
index 0000000000..267209cf03
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class tests disk balancer operations.
+ */
+public class TestDiskBalancer {
+
+ /**
+ * Set a timeout for each test.
+ */
+ @Rule
+ public Timeout timeout = Timeout.seconds(300);
+
+ private static ScmClient storageClient;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration ozoneConf;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ ozoneConf = new OzoneConfiguration();
+ ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
+ cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
+ storageClient = new ContainerOperationClient(ozoneConf);
+ cluster.waitForClusterToBeReady();
+
+ for (DatanodeDetails dn: cluster.getStorageContainerManager()
+ .getScmNodeManager().getAllNodes()) {
+ ((DatanodeInfo) dn).updateStorageReports(
+ HddsTestUtils.getRandomNodeReport(3, 1).getStorageReportList());
+ }
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ storageClient.getDiskBalancerReport(2);
+
+ assertEquals(2, reportProtoList.size());
+ Assert.assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ // TODO: Test status command with datanodes in balancing
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org