You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by yi...@apache.org on 2023/02/12 13:56:12 UTC
[ozone] branch HDDS-5713 updated (35791f59ba -> 6ae2da8218)
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a change to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 35791f59ba HDDS-7234. Add a common option for DiskBalancer commands (#3762)
omit 4a0132ed4b HDDS-7205. DiskBalancer CLI (#3739)
omit b23901827f HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
omit 09a0b8e1b7 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
add 3cf882e93c HDDS-6243. sourceToTargetMap in ContainerBalancer doesn't support multiple entries with same source. (#3485)
add 20c45aee39 HDDS-7161. Make Checksum.int2ByteString() zero-copy. (#3709)
add 9333f1e5bf HDDS-7166. Memory leak in Recon when replacing DB for new checkpoint (#3713)
add e1f4548d35 HDDS-7173. Fix TestReconOmMetadataManagerImpl#testUpdateOmDB (#3719)
add 1caf752269 HDDS-7172. Intermittent failure in Close pipeline smoketest (#3717)
add 93ae1cfcd5 HDDS-7132. GetFileStatus returns NULL for some paths. (#3687)
add f8ca8c5aff HDDS-6967. EC: Handle maintenance replicas in ECUnderReplicationHandler (#3661)
add 4f3e767b69 HDDS-6493. Improve TestOmKeyInfoCodec error handling (#3718)
add 2f757f6287 HDDS-7165. Remove AbstractContractUnbufferTest (#3714)
add 4585bbe89d HDDS-2954. Support admin groups (#3693)
add 48230b6fe0 HDDS-7158. ldb cli command supports to scan container V3. (#3705)
add 3dcd68ca3d HDDS-6621. Include annotation processor module in regular project hierarchy (#3326)
add 09e89fd67e HDDS-6312. Use KeyPrefixContainer table to accelerate the process of DELETE/UPDATE events (#3082)
add 537bac8887 HDDS-7176. Sort the container replica according to the replica index (#3720)
add b7c13dba3c HDDS-7163. Ozone debug container CLI supports container export. (#3715)
add 3e0cb27632 HDDS-6975. EC: Define the value of Maintenance Redundancy for EC containers (#3723)
add 919e7755ab HDDS-7186. Add support for Rocksdb user info LOG configure. (#3731)
add b093984f16 HDDS-7175. Timed out tests missing from bundle (#3721)
add abda1f2786 HDDS-7125. Inaccurate numBlocksDeletion when getTransactions (#3681)
add 492ae919a2 HDDS-7171. Add SupportedSourceVersion in annotation processors (#3724)
add a8efd9e6a7 HDDS-7200. Fixed inaccurate get request status code (#3733)
add 3d3e9eaf00 HDDS-7197. Schema V3 RocksDB instance statitics register as metrics collision (#3734)
add ec9327de59 HDDS-7201. testContainerIsReplicatedWhenAllNodesGotoMaintenance is failing frequently (#3737)
add 2ce64319a4 HDDS-7143. Recon: Fix left to right scrolling on Pipelines page (#3735)
add ab4288ccb8 HDDS-7191. Separate prop for s3 admin (#3732)
add 25dc267115 HDDS-7086. DataNode UI to show failed volumes too (#3740)
add e84aa4c4ea HDDS-2476. Share more code between metadata and data scanners. (#3727)
add 8f32c7e05a HDDS-7174. Migrate SCM integration tests to JUnit 5 (#3722)
add 4030265b0c HDDS-7204. EC: Schedule UnderReplicatedProcessor and OverReplicatedProcessor threads in RM instead of StorageContainerManager (#3738)
add 15fcb733c7 HDDS-7209. Bump Jersey2 to 2.34 (#3745)
add 64ec833e2b HDDS-7192. EC: ReplicationManager - create handlers to perform various container checks (#3743)
add 9ad75b2300 HDDS-7114. CHAINED and STAND_ALONE replication-type should be removed from help message (#3672)
add 286dffa8fa HDDS-7193. S3gateway Grpc Ranger Authorizer NPE Unhandled Exception (#3749)
add 3130b03846 HDDS-7144. Recon: Make only hostname fixed in Datanodes page. (#3736)
add 1dd5315704 HDDS-7225. Add Ratis tests for HealthCheck handlers of Replication Manager (#3757)
add 1cf5678224 HDDS-7207. Fixed typo "layload" in om-echo command description as payload (#3756)
add 1107fe711c HDDS-7221. EC: ReplicationManager - Encapsulate the under and over rep queues into a queue object (#3758)
add 9d3bd02215 HDDS-7185. Encode asterisk for parsing signature (#3726)
add d9b92e7a33 HDDS-7203. LookupKey Latency breakdown (#3742)
add 1a6b4d2d6f HDDS-7239. EC: Add a Handler for CLOSING containers in Replication Manager (#3769)
add 6c1a5ee07e HDDS-7240. List all volume operation should go through ACL check as well in order to trigger audit logging (#3770)
add 04efea814a HDDS-7251. Replace Log4j 1.x with Reload4j (#3773)
add f6f89a021c HDDS-7075. Implement FileSystem listStatusIterator to support file list paging (#3711)
add c7038e08a8 HDDS-7039. EC: Handle the placement policy check in ECUnderReplicationHandler (#3645)
add a8df5c530b HDDS-7145. Recon: Show last OM DB sync time on Overview page (#3765)
add 69695eabb7 HDDS-6492. Add metric for failed container moves (#3751)
add d22e42a2ac HDDS-7160. Recon: Make Container and Bucket stats clickable in Overviews page (#3775)
add cb886035ac HDDS-7257. Bump snakeyaml to 1.32 (#3776)
add 55efab1b3e HDDS-6863. Add Group-Id & Ratis-Roles Information for OM UI (#3520)
add 718f7309a8 HDDS-7222. Recon Inactive pipelines not shown. (#3753)
add efb3a80f73 HDDS-7232. Introduce container-level lock while handling container report events by SCM and Recon (#3766)
add c96a342448 HDDS-7263. Add a handler for Quasi Closed containers to RM (#3785)
add 1ff97328b7 HDDS-7206. Change the placement policy interface to allow existing nodes to be specified for Rack Scatter Policy. (#3744)
add cdd0fb4ce1 HDDS-7229. Introduce container location cache in ScmClient (#3771)
add 8dbbb4eb5f HDDS-7268. EC: Fix tests for HealthCheck handlers of RM that use Replica Indexes for Ratis Containers (#3793)
add d029910e75 HDDS-7283. Set coreSize equal to maxSize for threadPoolExecutor (#3792)
add 649d713817 HDDS-7235. EOFException occurs when executing TPC-DS using o3fs (#3764)
add 3b6ae4f761 HDDS-7285. Fix concurrency issue in DeleteBlocksCommandHandler (#3796)
add 23a0f743bc HDDS-7219. Remove not applicable info from ozone bucket list for links (#3777)
add 1585f660c1 HDDS-7250. Add HttpServer metrics (#3772)
add 0aabe8abb6 HDDS-7282. Add EstimatedKeyCount metrics for SCM DB (#3791)
add 4e364001ac HDDS-7211. Remove dead code from KeyManagerImpl (#3747)
add 0edb2ecfe4 HDDS-7217. OM logs wrong bucket layout (#3750)
add 2737d3d065 HDDS-7208. Erasure coding and encryption are not flagged on FileStatus (#3768)
add e0ea7df9e5 HDDS-7259. Fix uncounted blocksDeleted in BlockDeletingService (#3779)
add 099422c5db HDDS-7293. Bump jackson2 to 2.13.4 (#3808)
add 625e84f95e HDDS-7286. Clean up ContainerManager (#3797)
add e7f4c05d09 HDDS-7294. Move ozone manager background services to a separate package (#3805)
add 69d4dbd95a HDDS-7302. Fix inaccurate container block count caused by missing block (#3814)
add 17753bc93c HDDS-7287. Send deleteBlocksRequest with correct retry count (#3799)
add b808deb8c8 HDDS-6664. Implements getUri in TrashOzoneFileSystem (#3795)
add 32df39e8c8 HDDS-7299. Migrate simple tests in hdds-container-service to JUnit5 (#3813)
add 834ef98018 HDDS-7230. Implement GetKeyInfo API (#3780)
add f0040c294b HDDS-7304. EC: EC Decode can fail when byteBuffer from elastic pool is larger than chunksize (#3817)
add afb47176b7 HDDS-7271. Ozone Integration test shows memory leak (graceful shutdown cleanup) (#3787)
add bb46766e23 Revert "HDDS-7271. Ozone Integration test shows memory leak (graceful shutdown cleanup) (#3787)"
add 74deb5959a HDDS-7236. Enable Recon SCM DB bootstrap by default. (#3823)
add 0a5fc405e5 HDDS-6440. Handle custom metadata (x-amz-meta) during put-object through S3 API. (#3728)
add e529ed4586 HDDS-7088. OM incorrectly detects SCM Ratis Group ID when OM and SCM are colocated with same Ratis storage directory. (#3809)
add 54af0dacb0 HDDS-7313. Update github actions for Node16 (#3825)
add 15217fe59a HDDS-7271. Ozone Integration test shows memory leak (graceful shutdown cleanup) (#3826)
add a905b4bbb6 HDDS-6946. Bump Hadoop to 3.3.4 (#3557)
add 2bfb6d9ec5 HDDS-7315 SCM order of close of dbstore and other services (#3830)
add b18fb595e4 HDDS-7308 Removed cluster2 from serviceIds (#3819)
add 4515fe20c2 HDDS-7291. Fixing exception handling in case of non positive replica index (#3806)
add 58f0749b52 HDDS-7270. Fix bug in checking healthy replica placement policy check in EC underReplication handler (#3828)
add 6e2de99324 HDDS-7308. Fix config example syntax highlight in OM HA (#3833)
add 330b2e308d HDDS-7322. Remove unused dependencies hsqldb and jettison (#3834)
add 46e58a6350 HDDS-7300. Race condition between full data scan and block deletion (#3811)
add 02341d77f3 HDDS-7269. Remove fair lock configuration for container state management flows. (#3815)
add 35eedb6b36 HDDS-7303. EC: ECBlockReconstructedStripeInputStream should set initialized only at the end of init() (#3816)
add c45b31a4f4 HDDS-7330. Cleanup of BucketManagerImpl (#3838)
add c0de6bcdeb HDDS-7261. Add container location cache metrics. (#3829)
add 554eb9eb54 HDDS-7337. Replace set-output in Github Actions workflow (#3839)
add 9d05818a87 HDDS-7343. Do not log exception on file not found in getFileStatus() (#3849)
add 2dc800dc80 HDDS-7214. Continuous start & stop can have hanging threads in stopping (#3782)
add f7ba4ba588 HDDS-7340. Bump jackson-databind to 2.13.4.2 (#3845)
add 02c266d3fe HDDS-6893. EC: ReplicationManager - move the empty container handling into RM from Legacy (#3831)
add 11f4686ce8 HDDS-7254. Document that moving SCM from non-HA to HA is currently unsupported. (#3846)
add 03e670b865 HDDS-7149. Update ratis version to 2.4.0 and thirdparty version to 1.0.2. (#3855)
add a6316c8320 HDDS-7352. OM log flooded by AWSV4AuthValidator (#3857)
add 237a9a1594 HDDS-7058. EC: ReplicationManager - Implement ratis container replication check handler (#3802)
add 977ab59e11 HDDS-7341. EC: Close pipelines with unregistered nodes (#3850)
add d5dc65eaa0 HDDS-7305. Fix Hadoop imports (#3822)
add d32e96c835 HDDS-7351. Use jackson-bom to ensure consistent Jackson version (#3856)
add e45f9b8333 HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)
add ff6d15f5df HDDS-7354. SchemaV3 blockData not deleted in table (#3860)
add 3fd7cd0896 Revert "HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)"
add 1fa6d02e8a HDDS-6930. SCM,OM,RECON should not print ERROR and exit with code 1 on successful shutdown (#3848)
add df48ca4fe3 HDDS-7356. Update SCM-HA.zh.md to match the English version (#3861)
add 31560fcdfe HDDS-7355. non-primordial scm fail to get signed cert from primordial SCM when converting an unsecure cluster to secure (#3859)
add f9b74a25cf HDDS-6210. EC: Add EC metrics (#3851)
add 5c2a39365a HDDS-7369. Fix wrong order of command arguments in Nonrolling-Upgrade.md (#3866)
add 13a6d0178e HDDS-7141. Recon: Improve Disk Usage Page (#3789)
add ae59f8ae5f HDDS-7248. Recon: Expand the container status page to show all unhealthy container states (#3837)
add ecdfc20756 HDDS-7199. Implement new mix workload Read/Write Freon command (#3872)
add fdc57a93cb HDDS-7403. README Security Improvement (#3879)
add df0d1e81e3 HDDS-7368. [Multi-Tenant] Add Volume Existence check in preExecute for OMTenantCreateRequest (#3869)
add 2f1117539c HDDS-7284. JVM crash for rocksdb for read/write after close (#3801)
add 74aef20c9e HDDS-7182. Add property to control RocksDB max open files (#3843)
add b9a47f6bbf HDDS-7253. Fix exception when '/' in key name (#3774)
add 340f3a7876 HDDS-7381. Cleanup of VolumeManagerImpl (#3873)
add dfc13a0506 HDDS-7258. Cleanup the allocated but uncommitted blocks (#3778)
add 5b7f448a53 HDDS-7121. Support namespace summaries (du, dist & counts) for legacy FS buckets (#3746)
add 462f32dd69 HDDS-7396. Force close non-RATIS containers in ReplicationManager (#3877)
add 561788ebe7 Revert "HDDS-7253. Fix exception when '/' in key name (#3774)"
add 9449747ff7 HDDS-7413. Fix logging while marking container state unhealthy (#3887)
add a664ccac6a HDDS-7342. Move encryption-related code from MultipartCryptoKeyInputStream to OzoneCryptoInputStream (#3852)
add b5ecea61de HDDS-7349. Flaky integration test have memory leak for RatisDropwizardExports (#3858)
add 0652ba433b HDDS-7422. Bump woodstox-core from 5.0.3 to 5.4.0 (#3886)
add 557d7f84ff HDDS-7402. Adapt CommandQueue to track the count of each queued command type (#3891)
add 3fb0dfb464 HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)
add 901bcf2dcc HDDS-7424. Bump jetty to 9.4.49.v20220914 (#3894)
add a84baced7e HDDS-7220. SCM should use sub-ca certificate for token signature without HA enabled (#3752)
add 880f87cd4e HDDS-7370. Add pending commands in SCM to Datanode command count (#3867)
add f6cad7c55f HDDS-7328. Improve Deletion of FSO Paths (#3844)
add dd67faa0b0 HDDS-7406. Remove unused the exception & improve debug log in KeyDeletingService (#3880)
add 3a52215bb7 HDDS-1157. TestOzoneContainerWithTLS is failing with SSLHandshakeException (#3888)
add f0d41c5fcb HDDS-7316. Print stacktrace to identify the location of RocksObject leaks. (#3827)
add bd8a1610b3 HDDS-7421. Respect OZONE_LOGLEVEL and OZONE_ROOT_LOGGER for CLI commands (#3892)
add 30d1a4e359 HDDS-7420. Bump Spring framework from 5.2.20 to 5.3.23 (#3902)
add cc574c6c5f HDDS-7432. Move command summary into Commands object in CommandQueue (#3901)
add 7394f2c2ae HDDS-7361. Add general metrics for queues in Datanode (#3863)
add c187a8ff65 HDDS-7384. EC: ReplicationManager - implement deleting container handler (#3881)
add 3294d28cc9 HDDS-7231. Integrate the GetKeyInfo API to key read flows (#3800)
add 416c575685 HDDS-7146. Recon: Add button to trigger OM DB Sync (Backend) (#3821)
add f98376b743 HDDS-7244. Fix multiple reports queued up from same DN and using up heap (#3783)
add 9c6d6a9da5 HDDS-7290. provide a config to increase the list batch size in OzoneFileSystem (#3853)
add 2e1de7b95f HDDS-7363. Changes to Ozone CLI to validate if the jars in classpath files are present on an install (#3864)
add 03595deac6 HDDS-7429. Remove Tag from SCMMetadataStoreMetrics (#3897)
add c71e1da783 HDDS-7320. Enable Container SchemaV3 by default. (#3841)
add c7f1c7c9ee HDDS-7447. Leak of TableIterator in DirectoryDeletingService. (#3922)
add 8dded6ce7e HDDS-6926. Add support for shaded protobufs used by hadoop-client/spark. (#3915)
add a77cc1da90 HDDS-7435. ContainerOperationClient cleanup (#3910)
add 074d225f2a HDDS-7025. Add cache metrics in OM. (#3865)
add 891e9f57a9 HDDS-7090. EC: delete empty closed EC container (#3660)
add d004f69b98 HDDS-6757. Test "size.leaving.source.max" limit in ContainerBalancer (#3904)
add b316e73095 HDDS-7423. DB Scanner should allow iteration from specific key (#3893)
add 5dd389b7cb HDDS-5866. Discrepancy in Trash directory in ofs vs o3fs. (#3906)
add 6fed1c9434 HDDS-7324. S3G: Avoid calling getS3VolumeContext by using GetGeyInfo API. (#3911)
add 2037968a56 HDDS-7419. Integrate the GetKeyInfo API to OFS (#3912)
add 3c0ec81687 HDDS-7362. Bring back the mechanism to share LRU cache between RocksDB in datanodes (#3921)
add 5453cd6aec HDDS-3419. Throw exception with correct code when available data nodes are not sufficient (#3847)
add 4ec46bc102 HDDS-7460. Bump snakeyaml from 1.32 to 1.33 (#3935)
add 715a3f6d06 Upgrade RocksDB to 7.7.3 (#3924)
add f95da6c66e HDDS-7451. Add UT of SyncDataFromOM for SequenceNumberNotFoundException (#3927)
add 14a1668521 HDDS-7326. Intermittent timeout in TestECContainerRecovery.testContainerRecoveryOverReplicationProcessing (#3941)
add a499123287 HDDS-6896. EC: ReplicationManager - Add relevant metrics to the various ReplicationManager classes (#3909)
add f9df0593e4 HDDS-5112. Recon Insights page does not list buckets when only one volume is present. (#3907)
add 436b4b21eb HDDS-2642. Expose decommission / maintenance metrics via JMX (#3781)
add 39dfe7ad86 HDDS-7453. Check certificate expiration at service startup, renew if necessary. (#3930)
add 0af369e25f HDDS-7440. Delete package-info class for org.apache.hadoop.io.erasurecode.rawcoder to avoid conflict with Hadoop (#3925)
add 3d3e6cb19a HDDS-7314. Recon graceful shutdown while closing RocksDB handle. (#3931)
add 9a25a8ea7e HDDS-7360. Investigate TestRootedOzoneFileSystem test cases. (#3928)
add 39747954a6 HDDS-7323. Recon: Auto refresh toggle is switched back when visiting new site (#3868)
add fa5695b677 HDDS-7456 Recon: Refreshed at not working on Pipeline and DataNode page (#3933)
add 03fa68109a HDDS-7408. Recon: Refactor container status page (#3916)
add ec0d24a9da HDDS-4385. Search function using container ID on Recon Missing Container (#3917)
add 4dc3083954 HDDS-7469. Recon: Add more information to Open Containers column on on the Datanodes page. (#3944)
add c4e9e348fa HDDS-7446. Cleanup possibly onDisk unreferenced blocks (#3923)
add a13c62b605 HDDS-7471. EC: Notify ReplicationManager when a heartbeat updates datanode command counts (#3945)
add a8a0a4533c HDDS-7475. Remove outdated Hadoop dependencies (#3950)
add b3e7f3c320 HDDS-7485. Bump commons-codec to 1.15 (#3955)
add 73d9082168 HDDS-7468. Recon: clusterState API Changes for adding missingContainerCount (#3952)
add 0f2f15b79f HDDS-7490. fix metaTable error (#3961)
add b46f961b59 HDDS-7255. Add metrics for container reports events (#3936)
add 006df10d67 HDDS-7101. EC: ReplicationManager - handle UNHEALTHY replicas (#3962)
add 8518ed1cff HDDS-6971. EC: Add EC block checksum computer. (#3790)
add 95568ab735 HDDS-7095. allow on demand scanning for containers (#3788)
add ba41ef5041 HDDS-7487. Support fine grained certificate lifetime for efficient test. (#3954)
add 7e38bfe164 HDDS-7488. EC: ReplicationManager: Move Mis-Replicated into a separate unhealthy state (#3956)
add 559ad868ef HDDS-4402. Recon dashboard page does not load until missing containers API returns data. (#3968)
add 412a13aa56 HDDS-7245. Recon: Add button to trigger OM DB Sync (Frontend) (#3929)
add 1bc9c8a0ac HDDS-7474. Recon: Add count to containers page tab label (#3951)
add d476389a4b HDDS-7260. dfs -du should return correct disk space consumed by all replicas (#3810)
add 6ff33075f8 HDDS-7470. [Quota] bucket is created crossing quota makes OM crash and unable to start (#3960)
add 6a0d4d5c7f HDDS-7346. Cannot set bucket args when the bucket has quota set (#3975)
add 85e7cd1867 HDDS-7498. Add permission check when `--user` is specified in `ozone sh volume list` (#3971)
add 5e23afee10 HDDS-4565. [FSO]Delete : quotaReleased on directory deletion should consider all sub paths (#3973)
add edefe79434 HDDS-7476. SCM root CA stored under hdds.datanode.dir if ozone.metadata.dirs is not specified (#3948)
add 0ed0240c66 HDDS-7280. New test addition to TestOMBucketCreateRequestWithFSO (#3794)
add 0f0645ed23 HDDS-5526. ContainerBalancer#checkConditionsForBalancing pre-emptively checks iteration limits. (#3895)
add 99493d52f8 HDDS-7518. Fix intermittent failure in ozonesecure replication test (#3986)
add 1e86860e1b HDDS-7462. EC: Fix Reconstruction Issue with StaleRecoveringContainerScrubbingService (#3939)
add cb9b5d06dd HDDS-7092. EC: Offline Recovery with simultaneous Over Replication & Under Replication (#3984)
add c8648b159e HDDS-7528. EC: ReplicationManager - refactor logic to send datanode commands into a central place (#3990)
add 1370de18b8 HDDS-7532. EC: ReplicationManager - remove calls to ECHealthCheck from under and over replication processing (#3995)
add 864397d1c6 HDDS-7187. EC: Retry failed writes before rewrite to a new block group (#3977)
add b92f7c58f5 HDDS-7481. Set quota with reduced size than actual used do not fail (#3978)
add 302097fabb HDDS-7353. Cleanup of unused code in SCM (#3905)
add 6abee8a48c HDDS-7534. Intermittent failure in TestKeyDeletingService#checkDeletionForEmptyKey (#3998)
add 7f3f28c54e HDDS-7538. Remove unused OmUtils#getMD5Digest (#3996)
add 4145cf5a08 HDDS-7449. Avoid overwriting bucket encryption properties when quota/replication config are set on an existing bucket (#3992)
add 6570c19081 HDDS-7535. Eliminate duplicated config in LegacyReplicationManager (#4000)
add 7af8d06d87 HDDS-7289. Bump protobuf-java from 3.19.2 to 3.19.6 (#4003)
add 27ff3b6442 HDDS-7486. Support KeyStoreFactory which supports keyManager and trustManager reload (#3959)
add e50c443d24 HDDS-7536. Remove unused dependency declarations (#4001)
add 8db02c864b HDDS-7500. Ozone recon server Memory Optimization. (#3974)
add f552cf21a1 HDDS-7540. Separate scheduled CI from push/PR workflows (#4004)
add 829dfce410 HDDS-7554. Recon UI should show DORMANT in pipeline status filter (#4010)
add eeeb0b548c HDDS-6615. EC: Improve write performance by pipelining encode and flush (#3994)
add f1d651d0b5 HDDS-7561. Improve setquota, clrquota CLI usage (#4016)
add 627c90c233 HDDS-7510. Recon: Return number of open containers in `/clusterState` endpoint (#3989)
add 4de7d56fc4 HDDS-7394. OM RPC FairCallQueue decay decision metrics list caller username in the metric (#3878)
add 9a5bfb4fbd HDDS-7568. Bump copyright year in NOTICE (#4018)
add 45778cc699 HDDS-7540. Let reusable workflow inherit secrets (#4012)
add cb9caca398 HDDS-7575. Correct showing of RATIS-THREE icon in Recon UI (#4026)
add fe86f3bb49 HDDS-7567. Invalid entries in LICENSE (#4020)
add 50a9d46d80 HDDS-7497. Fix mkdir does not update bucket's usedNamespace (#3969)
add be092c4153 HDDS-7563. Add a handler for under replicated Ratis containers in RM (#4025)
add 984d26a165 HDDS-7562. Suppress warning about long filenames in tar (#4017)
add 1119f9d949 HDDS-7579. Use Netty 4.1.77 for consistency (#4031)
add 069e038c4f HDDS-7253. Fix exception when '/' in key name (#4038)
add 47a5671cc5 HDDS-7581. Fix update-jar-report for snapshot (#4034)
add b532d2f7f5 HDDS-7541. FSO recursive delete directory with hierarchy takes much time for cleanup (#4008)
add ee2bd17440 HDDS-7555. Upgrade Ratis to 2.4.2-8b8bdda-SNAPSHOT. (#4028)
add a1bc9b86de HDDS-5704. Ozone URI syntax description in help content needs to mention about ozone service id (#3862)
add 037492f839 HDDS-7472. EC: Fix NSSummaryEndpoint#getDiskUsage for EC keys (#3987)
add 721b02593b HDDS-7526. Avoid overwriting replication config on existing bucket when quota is set (#4013)
add fd3ddc42b2 HDDS-7599. Bump commons-net from 3.6 to 3.9.0 (#4046)
add d50a9dd894 HDDS-7550. Recon: UI sorting not functioning as expected (#4009)
add 565775c7dd HDDS-7464. Container Report at SCM is not coming separately for ICR and FCR in prometheus endpoint (#4035)
add a93ca68066 HDDS-7597. Clarify SCM admin access denied message (#4056)
add 8967738e93 HDDS-7492. Extend Placement Policy Interface to select mis-replicated replicas to copy (#4006)
add ebd1c747ac HDDS-7615. Import Guava classes directly, not from hadoop-thirdparty (#4062)
add 77d386de5f HDDS-7502. Recon: Show number of open containers on the Overview page (#4043)
add 6eed1ec3d3 HDDS-7265. ScatterRackPolicy for Ratis pipeline provider container placement (#4002)
add 845ceff4d6 HDDS-7603. Enable Erasure coding and encryption flag on FileStatus for ofs (#4060)
add a9bb08889c HDDS-7587. Update NOTICE and LICENSE in license/bin/ (#4039)
add e7567f8958 HDDS-7617. Mark testECContainerRecoveryWithTimedOutRecovery as flaky
add 66e277386e HDDS-7572. Use keyManager and trustManager provided by keyStoreFactory in datanode grpc services (#4032)
add 76fb0163df HDDS-7580. Add option to show key count in DBScanner. (#4033)
add d0e6824eec HDDS-7614. Add subscription mechanism to ContainerReplicaPendingOps (#4064)
add c9a0e2cd10 HDDS-7589. Unify multipart stream reading. (#4041)
add 3fe4b4e156 HDDS-7595. Add OmId in OM's UI (#4047)
add e4a1993e79 HDDS-7618. Replication Commands should timeout if not processed on datanodes in time (#4069)
add a6307d71d0 HDDS-7628. Disable flaky TestOzoneContainerWithTLS#testContainerDownload
add 4c529ebc11 HDDS-7591. Add a handler for over replicated Ratis containers to RM (#4055)
add d6f63bf2e5 HDDS-7619. Update SCM term in datanode when command is received (#4072)
add 2279689dd3 HDDS-7008. Mark TestOzoneHARatisLogParser as flaky (#4078)
add 551ef3c020 HDDS-7640. EC: UNHEALTHY replicas not replaced by healthy replicas from a CLOSED container by RM (#4083)
add 2fc94fa05b HDDS-7582. ECUnderReplicationHandler does not consider pending adds when finding targets (#4085)
add 6064b84576 HDDS-7503. CLI admin namespace -quota should return whole file system disk capacity consumed by all replicas (#3976)
add 2ba8bb71f1 HDDS-7643. rocksdb iterator is not closed properly (#4084)
add a756937a16 HDDS-7627. EC: Bug fix for calculating Misreplication Count (#4088)
add 5eead92666 HDDS-7592. Recursive delete of directory multiple time impacts some directory not removed (#4068)
add fd0eddda8c HDDS-7642. EC: Add debug logging to the Replication Manager check handlers (#4087)
add b347fde11b HDDS-7628. Disable flaky TestOzoneContainerWithTLS#testCreateOzoneContainer
add 2d4bc6a3e2 HDDS-7636. Remove hadoop security dependency in org.apache.hadoop.hdds.security.ssl package (#4080)
add 1a5e698c6b HDDS-7521. EC: Misreplication Handler changes for Placement Policy interface changes (#4061)
add 1f7e9cc8ed HDDS-7620. Check term for async commands before processing (#4096)
add 517e3e15c5 HDDS-7371. Create properties for all dependency versions (#4059)
add fdf219f179 HDDS-5366. [Ozone-Streaming] Implement stream method to ContainerStateMachine. (#2358). Contributed by mingchao zhao
add 0afbbc541d HDDS-5452. Add link method to ContainerStateMachine for Ratis streaming (#2422)
add 9eafca6ee8 HDDS-5481. Fix stream() and link() method in ContainerStateMachine. (#2451)
add da95602eba HDDS-5480. [Ozone-Streaming] Client and server should support stream setup. (#2452)
add 82153a26ed HDDS-5488. [Ozone-Streaming] Add a new BlockOutputStream/KeyOutputStream to support streaming api (#2495)
add 5a294b7d54 HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer to avoid buffer copying (#2557)
add ff59eeb27f HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to ByteBufferStreamOutput (#2603)
add f27388a5cc HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
add 94f87891a7 HDDS-5486. [Ozone-Streaming] Streaming supports writing in Pipline mode (#2682)
add 9d01c02c98 HDDS-5849. [Ozone-Streaming]Write exceptions occur after checksum is enabled (#2729)
add 122f96c029 HDDS-5674.[Ozone-Streaming] Handle client retries on exception (#2701)
add e063706ed3 HDDS-5895. [Ozone-Streaming] Make raft.server.data-stream.client.pool.size configurable (#2766)
add 75efaae8cf HDDS-5763. Provide an Executor for each LocalStream in ContainerStateMachine (#2782)
add 2e86e93321 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream config (#2841)
add 9a7edb0240 HDDS-5961. [Ozone-Streaming] update the usage space of Containers in the stream write (#2833)
add 237e0afed3 HDDS-5879. [Ozone-Streaming] OzoneBucket add the createMultipartStreamKey method (#2760)
add 95e317b551 HDDS-5743. [Ozone-Streaming] Add option to write files via streaming api in ofs and o3fs. (#2770)
add 9a2de9a530 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. (#2866)
add ac9921d65d HDDS-6039. Define a minimum packet size during streaming writes. (#2883)
add a67569fe83 HDDS-6130. [Ozone-Streaming] When releaseBuffers will get “Couldn 't find the required future” (#2939)
add f85b42dcee HDDS-6139. [Ozone-Streaming] Fix incorrect computation of totalAckDataLength. (#2978)
add 0e8c8004a1 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
add 2657f5ad95 HDDS-6138.[Ozone-Streaming] Define a limit on the size of the retry bufferList. (#2946)
add 14ffbd9cb1 HDDS-6298. Add XceiverServerRatis stream config (#3070)
add 872e3061fe HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support FlushDelay. (#3002)
add 92054a4d2f HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE (#3060)
add 8a5f2d6192 HDDS-6229. [Ozone-Streaming] Data Channel abstraction on datanode (#3023)
add 8f68101eb8 HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
add bb3af31757 HDDS-6388. [Ozone-Streaming] Streaming write support both pipeline model and star model (#3145)
add dfefdf20eb HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters. (#3207)
add 95dbea64a0 HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput. (#3195)
add 6a6384233e HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at the end of the stream. (#3229)
add 3d821e31ff HDDS-5666. Add option to createKey via streaming api in Freon (#2574)
add 7f22916889 HDDS-6592. [Ozone-Streaming] Fix ContainerStateMachine#applyTransaction assert error (#3315)
add 45954da28b HDDS-6842. [Ozone-Streaming] Reduce the number of watch requests in StreamCommitWatcher. (#3492)
add 7184025ffb HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use streaming to put EC key. (#3516)
add 07bf1dbcf0 HDDS-6955. [Ozone-streaming] Add explicit stream flag in ozone shell (#3559)
add 13da2a7c4a HDDS-7431. [Ozone-Streaming] Disable data steam by default. (#3900)
add 8a0e347105 HDDS-7438. [Ozone-Streaming] Add a createStreamKey method to OzoneBucket. (#3914)
add 9bbf5a1ada HDDS-7425. Add documentation for the new Streaming Pipeline feature. (#3913)
add 1e155f3820 HDDS-7478. [Ozone-Streaming] NPE in when creating a file with o3fs. (#3949)
add 0650ad9b08 HDDS-7426. Add a new acceptance test for Streaming Pipeline. (#4019)
add 8b2ca00822 HDDS-7616. EC: Refactor Unhealthy Replicated Processor (#4063)
add 886733b373 HDDS-7605. Improve logging in Container Balancer (#4067)
add ca2d59bde4 HDDS-6650. S3MultipartUpload support update bucket usedNamespace. (#4081)
add 2ac31e1bcb HDDS-7398. Tool to remove old certs from the scm db (#3972)
add 1235e9b3d1 HDDS-7399. Enable specifying external root ca (#4053)
add d93dcd396f HDDS-7649. S3 multipart upload EC release space quota wrong for old version (#4095)
add d5f917274c HDDS-7621. Update SCM term in datanode from heartbeat without any commands (#4101)
add ef480765d8 HDDS-7654. EC: ReplicationManager - merge mis-rep queue into under replicated queue (#4099)
add 7ca9b5dfa6 HDDS-7565. FSO purge directory for old bucket can update quota for new bucket (#4021)
add bdeceeedad HDDS-7635. Update failure metrics when allocate block fails in preExecute. (#4086)
add 204f8e18c5 HDDS-7683. EC: ReplicationManager - UnderRep maintenance handler should not request nodes if none needed (#4109)
add c7785fa4fd HDDS-7463. SCM Pipeline scrubber never able to cleanup allocated pipeline. (#4093)
add 9077cb83d1 HDDS-7473. Ratis integration for support of remove registeration (#4112)
add add8f8bb16 HDDS-7686. Cherry-pick proto.lock files change from ozone-1.3 release branch to master (#4113)
add 157c591b13 HDDS-7631. Log format error on quotas when exceeding the space quota (#4076)
add fa8494cd1d HDDS-5331. Recon: Trigger PipelineSyncTask when DN becomes stale and ContainerHealthTask when DN becomes dead. (#4011)
add 58283b6230 HDDS-7684. Embed Matomo Web Analytics tracking code in docs. (#4111)
add c40cb07053 HDDS-7576. Prometheus metrics do not remove stale metrics until restart (#4057)
add a7f06c1964 HDDS-3486. Recon cannot track missing containers that were created and went missing while it is down. (#3947)
add b167d99d73 HDDS-7692. Remove usage of MonotonicClock and use default system clock instead (#4121)
add b828881f78 HDDS-7661. Ratis Misreplication Handler (#4100)
add bfc37bb886 HDDS-7641. Namespace quota validation is not present in multiple places (#4082)
add 98eb2ebd99 HDDS-7644. S3 multipart upload does not update quota namespace for missing parents (#4091)
add a89ba1a4c4 HDDS-7584. Addition of new OM node expels itself from the Ratis ring after restart (#4074)
add 7cfaf96b81 HDDS-7525. Migrate key digest from MD5 to SHA256 in Ozone shell
add 4184499e80 HDDS-7623. Do not compress container re-replication traffic by default (#4089)
add b352ad03ab HDDS-7409 [doc] Update documents for better presentation (#3884)
add 1c830c5bc7 HDDS-7564. Update Ozone version after 1.3.0 release (#4115)
add eb031df2ed HDDS-7648. Add a servername tag in UGI metrics. (#4094)
add 04f68c96f4 HDDS-7633. Compile error with Java 11: package com.sun.jmx.mbeanserver is not visible (#4077)
add c119557241 HDDS-7588. Intermittent failure in TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4040)
add 58da8a90a1 HDDS-7716. Log read requests rejected with permission denied in OM audit (#4136)
add 21717321c6 HDDS-7714. Docker cluster ozone-om-ha fails during docker-compose up (#4137)
add 9b48d8b1c0 HDDS-7696. MisReplicationHandler does not consider QUASI_CLOSED replicas as sources (#4144)
add 41f048804a HDDS-7339. Implement Certificate renewal task for services (#3982)
add 10811c597f HDDS-7666. EC: Unrecoverable EC containers with some remaining replicas may block decommissioning (#4118)
add 9d30c9a7ce HDDS-7606. Add SCM HA support in intellij run (#4058)
add 7d6ee034d7 HDDS-7727. EC: SCM unregistered event handler for DatanodeCommandCountUpdated (#4147)
add ecc2f189e4 HDDS-7708. No check for certificate duration config scenarios. (#4149)
add f2f0afa9f6 HDDS-7243. Remove RequestFeatureValidator from echoRPC method which supports only ValidationCondition.OLDER_CLIENT_REQUESTS (#4051)
add 2eb5805b6c HDDS-7738. SCM terminates when adding container to a closed pipeline (#4154)
add e09f9a208a HDDS-7729. EC: ECContainerReplicaCount should handle pending delete of unhealthy replicas (#4146)
add 4abe983acf HDDS-7695. EC metrics related to replication commands don't add up (#4152)
add a1cb9d567d HDDS-7628. Intermittent failure in TestOzoneContainerWithTLS (#4142)
add dcfd621e15 HDDS-7652. Volume Quota not enforced during write when bucket quota is not set (#4124)
add 07b67597e8 HDDS-7739. EC: Increase the information in the RM sending command log message (#4153)
add a80c6b1c73 HDDS-7726. EC: Enhance datanode reconstruction log message (#4155)
add b6cda5162f HDDS-4539. Container Health Task should not run until Recon has reached steady state. (#4049)
add ed7c60cafd HDDS-7560. Placement Policy Interface changes to handle Overreplication (#4014)
add ab91e46247 HDDS-7638. Ozone client change to support HSync. (#4104)
add cbbddc6fdc HDDS-7630. Show detailed OMResponse when OzoneManagerDoubleBuffer terminates (#4075)
add dff829623b HDDS-7705. Fix OM Bootstrap request (#4160)
add 4531701c4d HDDS-7753. Simplify DatanodeDetails#toString to improve log messages (#4161)
add de53086f13 HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. (#4042)
add 7229b35958 HDDS-6447. Refine SCM handling of unhealthy container replicas. (#3920)
add d56acae8c8 HDDS-7760. Let ContainerDataConstructor extend SafeConstructor (#4162)
add 9d5cfd6934 HDDS-7083. Spread container-copy directories (#3648)
add 5f54cc60ba HDDS-5934. Add OM NodeID to VERSION file. (#4138)
add 2bd13c3737 HDDS-7751. Volume quota must have all bucket quota set (#4165)
add 80f544b704 HDDS-7570. Provide a shareable ofs://temp directory (#4027)
add 1a63207add HDDS-7775. EC: Exception encountered while deleting UNHEALTHY replica in Datanode (#4173)
add f9dfd73e99 HDDS-7761. EC: ReplicationManager - Use placementPolicy.replicasToRemoveToFixOverreplication in EC Over replication handler (#4166)
add 253e97c218 HDDS-7774. Update outdated Trash documentation (#4172)
add c00d3af98a HDDS-7722. FSO buckets fail to invalidate open file table cache when committing a key (#4156)
add 7923e25be4 HDDS-7441. Rename function names of retrieving metadata keys (#3918)
add 59938f90bf HDDS-7455. ClassCastException: OzoneTokenIdentifier cannot be cast to String (#4159)
add 8b9aad6a45 HDDS-7754. Download of container is failing with SSL/TLS error during re-replication (#4174)
add c61ee3ea3e HDDS-7787. GetChecksum for EC files can fail intermittently with IndexOutOfBounds exception (#4180)
add b405f84621 HDDS-7542. Refactor DefaultReplicationConfig (#4005)
add 0c32c60257 HDDS-7718. Bump Netty to 4.1.86 and gRPC to 1.51.1 (#4139)
add 3bc2b79764 HDDS-7788. Ratis OverReplicationHandler should exclude stale replicas (#4183)
add dbe9abc670 HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed (#4179)
add d3f232396a HDDS-7748. Rename OMFileRequest.addToOpenFileTable() to avoid misuse. (#4176)
add be408031e3 HDDS-7804. UNHEALTHY replicas will not contribute to sufficient replication in RatisContainerReplicaCount (#4192)
add 6ff310eedd HDDS-7755. Ensure that acquired locks are always released. (#4191)
add 099cd1d680 HDDS-7807. TarContainerPacker closes streams multiple times (#4193)
add c6c36d4bcb HDDS-7801. Bucket not found when calling getKeyInfo with tenant context (#4189)
add 23a91637ff HDDS-7625. Do not compress OM/SCM checkpoints (#4130)
add 5ea64550b5 HDDS-7813. Handle Mismatched Replicas (OPEN or CLOSING) of QUASI-CLOSED containers in RM (#4195)
add 61a2e6e4b8 HDDS-7097. Container scanner log output lacks useful information (#4169)
add c25d38faf8 HDDS-7792. Fix package name typos in o.a.h.hdds.security.x509 (#4184)
add aed64de760 HDDS-7777. Implement container replication in push model (#4197)
add 151ee0afd8 HDDS-7820. Store container descriptor at start of tar archive (#4200)
add f1e94fc801 HDDS-7825. Warn when EC write exception occurs (#4201)
add 43b01d1bc5 HDDS-7833. EC: Refactor ReplicationSupervisor to allow Replication and ECReconstruction tasks (#4202)
add 08d8521060 HDDS-7822. Allow multiple commands per datanode in UnhealthyReplicationHandler (#4203)
add 0217fa4e8e HDDS-7834. Remove unnecessary streams (#4208)
add 4c6c1218a7 HDDS-7818. Modify Ratis Replication Handling in the new RM (#4204)
add e806fd6ce8 HDDS-7496. Make default bucket layout FSO in a backwards compatible way. (#3966)
add 17549db4a1 HDDS-7841. EC: Remove ECReconstructionSupervisor and send reconstruction commands to ReplicationSupervisor (#4210)
add ec3813fada HDDS-7808. Intermittent failure in TestReplicationManager#testUnderReplicationQueuePopulated (#4213)
add ec1e098694 HDDS-7372. Fix missing jars in classpath by specifying jar versions (#4097)
add 75a28b2e4c HDDS-7806. Add unit tests for push replication (#4216)
add 38733cfff1 HDDS-7770. Recon namespace summary endpoint to carry basic entity information as well (#4170)
add 9d4d358654 HDDS-7778. Add metrics for push replication (#4217)
add 4eddb9d2ac HDDS-6847. Audit for ALLOCATE_BLOCK should indicate number of allocated blocks (#3518)
add 3393354472 HDDS-7856. Disable flaky TestPushReplicator until fixed
add b16342af26 HDDS-7844. EC: Add normal and low priority to replication supervisor and commands (#4220)
add 6389d01605 HDDS-7856. Fix timeout in TestPushReplicator (#4225)
add 3084e8d41e HDDS-7859. ICR processing does not remove container reference in NodeManager for a deleted replica (#4228)
add 1a796f1b54 HDDS-7821. Let push replication use compression from configuration (#4229)
add 2410551d2e HDDS-6849. OM changes for tracking Snapshot Metadata. (#3582)
add a0a9d221d0 Merge branch 'master' into HDDS-6517-Snapshot
add 4dba9020ed HDDS-6853. [Snapshot] Implement Create Snapshot API. (#3652)
add eafd2ccec0 HDDS-6964. [Snapshot] Split out shared "Path based access" code from OM. (#3653)
add 03dfa243ac HDDS-7224. Create a new RocksDBCheckpoint Diff utility. (#3755)
add 6eb19a7710 HDDS-6855. [SNAPSHOTS] Path Based Access for Ozone Snapshots. (#3729)
add 4be3864675 Merge branch 'master' into HDDS-6517-Snapshot
add f30c0621ea HDDS-6983. Snapshot Chain - list of snapshots per snapshottable bucket (#3658)
add 9538954e4b HDDS-7427. [Snapshot] Add unit-testcases for Ozone snapshot feature. (#3896)
add e642ddeb55 HDDS-7281. [Snapshot] Handle RocksDB compaction DAG persistence and reconstruction (#3824)
add d1a834ce09 HDDS-7452. [snapshot] Add unit-testcases for snapshot create validation. (#3946)
add f1f1c0ba42 HDDS-7375. [Snapshot] Implement `ozone fs` command for creating snapshot (#3937)
add 83072e0f8d HDDS-7410. Ozone snapshot diff skeleton code. (#3885)
add 06ae2fe8d5 HDDS-7494. [Snapshot] Fix SnapshotInfo#dbTxSequenceNumber (de)serialization (#3965)
add 67e67250bd HDDS-6854. [Snapshot] Implement List Snapshot API (#3784)
add 6fe2ab8011 HDDS-7279. Snapshot Create requires Double Buffer Flush thread to split the commit batch. (#3958)
add 1a52ae199a HDDS-7571. [snapshot] Add unit-testcases for Ozone Snapshot create API name validation. (#4023)
add d96f207f3a HDDS-7512. [snapshot] List Snapshot returns an empty list for a non-existent bucket (#3993)
add f77dfa6f59 HDDS-6962. [Snapshot] Background Service to delete irrelevant SST files in a snapshot. (#3883)
add e5d9c38f6f HDDS-7466. SnapshotDiff Client Code. (#3942)
add 2554ec26d7 HDDS-7508. [Snapshot] Finish TestOMSnapshotDAG, restore CompactionNode SnapshotID field for debugging (#3981)
add 1ee083178d HDDS-7484. Snapshot ID on followers should match the one on the OM Leader (#3985)
add 113072c16c HDDS-7583. Support efficient Snapdiff only until configured Snapshot history (#4036)
add 7f8838070e HDDS-7647. [snapshot] Add unit-testcases for Ozone Snapshot List API (#4090)
add 96bb4369c0 Revert "HDDS-7647. [snapshot] Add unit-testcases for Ozone Snapshot List API (#4090)" (#4102)
add 39bd6090bd Revert "HDDS-7583. Support efficient Snapdiff only until configured Snapshot history (#4036)" (#4103)
add 3d289e9050 HDDS-7647. [snapshot] Add unit-testcases for Ozone Snapshot List API (#4105)
add 9fe6d106ac HDDS-7524. Compaction DAG node pruning (#4045)
add 125eccfef2 HDDS-7607. [Snapshot] SnapshotDiff command lists unmodified file as modified (#4108)
add 0bcd697107 HDDS-7690. [Snapshot] Use SST file list output from compaction DAG as SnapshotDiff input (#4119)
add d8765436c2 Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot
add 98e3a5033d Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot
add 52fb8ba492 HDDS-7690. Addendum: [Snapshot] Use SST file list output from compaction DAG as SnapshotDiff input (#4168)
add 52103f273a HDDS-6984. [Snapshot] Prevent bucket from being deleted if snapshot exists (#4015)
add 0c25be8cdf HDDS-7702. [snapshot] Add unit-testcases for Ozone fs createSnapshot (#4122)
add 26006d4fcf HDDS-7765. [Snapshot] Handle OzoneManager#getKeyInfo with OmMetadataReader. (#4185)
add 01762a385d HDDS-7506. [Snapshot] Expose more snapshot metrics under OMMetrics (#4164)
add c88dd820e1 HDDS-7779. [snapshot] Add unit-testcases for Ozone SnapshotDiff.
add 3e31e047d9 HDDS-7768. [Snapshot] Add acceptance tests (#4171)
add dae9285af3 HDDS-7704. [snapshot] Add unit-testcases for snapshot restore (#4148)
add 40688f637a HDDS-7819. [Snapshot] Add unit-testcases for fs delete of bucket havng snapshots (#4198)
add ae83ac54e0 HDDS-7600. [Snapshot] Mark all new proto message fields as optional (#4120)
add aea5edfe99 HDDS-7845. [Snapshot] Wait for RocksDB checkpoint directory creation (#4214)
add 106b193c2a Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot
add ccc814ee7f HDDS-6857. [Snapshot] Implement Snapshot Delete CLI and API (#4175)
add afa6030b90 HDDS-7861. [Snapshot] Delete keys from the source bucket to fix the flakiness of snapshot restore tests (#4230)
add f01cebe19b Merge remote-tracking branch 'asf/HDDS-6517-Snapshot'
add 2eae7cd575 HDDS-7826. Add support for application to probe output stream capability (#4205)
add b44a77db99 HDDS-7874. Disable flaky unit test: TestHddsSecureDatanodeInit.testCertificateRotation
add ab721e8ceb HDDS-7850. Bump maven-enforcer-plugin version to 3.2.1 (#4218)
add 22f38d6f0c HDDS-5626. Mark parts of TestAddRemoveOzoneManager, TestBlockOutputStreamFlushDelay and TestFailureHandlingByClient as flaky (#4240)
add a352ab9fcd HDDS-7688. Ozone Manager change to support HSync. (#4211)
add 583fdb8e34 HDDS-7879. Intermittent BindException in HA integration tests (#4241)
add 8741083374 HDDS-7868. Intermittent failure in TestOmAcls (#4239)
add fd32156178 HDDS-4303. Ensure LayoutVersion is unique and monotonicaly increasing in LayoutFeature enums (#4224)
add 9fba172b3f HDDS-7873. [Snapshot] Prune backup SST files that can be expanded periodically (#4235)
add 3732f55c16 HDDS-7277. [Snapshot] Implement snapshot key listing in Ozone Shell (#4125)
add 6a66db0a99 HDDS-7886. Mark parts of TestRatisPipelineCreateAndDestroy as flaky
add 15cc89b602 HDDS-3591. ozone.administrators support dynamic changes through the cli. (#4221)
add c6c58ab736 HDDS-7450. Improve bucket creation message (#3997)
add 4ded8faff3 HDDS-7881. Intermittent timeout in basic acceptance test (#4243)
add 19e6ac9fb8 HDDS-7783. Add compile platform info in 'ozone version' command output (#4177)
add 406aef41e0 HDDS-7747. Document jq filtering examples for CLI responses. (#4157)
add 0a377c685a HDDS-7838. gRPC channel created block input/output stream not shutdown properly (#4215)
add fda247cf51 HDDS-7306. DeleteBlock retry count increased at incorrect place (#3818)
add 2e49013db4 HDDS-7091. Filter containers with pending deletion blocks before checking policy (#3655)
add 6c44169771 HDDS-7863. Remove OzoneStreamCapabilities (#4247)
add 940d678ef8 HDDS-7578. Wrong directory config used for SCM HA Ratis Snapshot (#4030)
add db1ea10645 HDDS-7126. Clean deletedBlock records of deleted containers (#3683)
add 471f1b2a93 HDDS-7082. Delete out of date audit logs (#3647)
add 9882cf7390 HDDS-7382. FSO bucket `rename` supports change modification time of parent dir (#3953)
add 0530b6dc88 HDDS-7854. Recon NPE when trace enabled (#4222)
add 3ca0a982dc HDDS-7851. Apply flush and compact on all column families (#4219)
add 9df3916614 HDDS-7864. Add integration test for replication (#4238)
add db896cd2ae HDDS-3265. Intermittent timeout in TestRatisPipelineLeader (#4248)
add 19467c4987 HDDS-7764. Potential discrepancy of key creation time may cause premature open key clean up. (#4242)
add 3e682b8470 HDDS-7489. Create a Freon tool to simulate datanodes (#4126)
add dba3ef9463 HDDS-7617. Mark TestECContainerRecovery as flaky
add 07629324c0 HDDS-6444. Apply HDDS-5875 to OMKeyCommitRequestWithFSO (#3250)
add 8007630491 HDDS-7857. Apply custom RocksDB log configuration for all schema versions (#4223)
add 1db8b735c4 HDDS-7321. Auto rocksDB small sst files compaction. (#3908)
add 11fd5b12fd HDDS-7917. EC: ECBlockInputStream should try spare replicas on error (#4253)
add 10a1fa6158 HDDS-7921. Migrate TestKeyManagerImpl to JUnit5 (#4257)
add 2250a481d6 HDDS-7799. Add container count to datanode usage info (#4209)
add 5daca052cc HDDS-6056. Recon /containers endpoint should return SCM container data instead of OM container data. (#4158)
add 5952e4ea80 HDDS-7871. Fix false positive in KeyManagerImpl#createFakeDirIfShould() (#4236)
add 3f7ded2a34 HDDS-7915. Force close QUASI_CLOSED replicas only when the container is CLOSED in Legacy RM (#4259)
add d1319d8187 HDDS-7924. Remove non-conflicting transitive-only dependency versions (#4261)
add 1bb74438fa HDDS-7815. Extract S3 secret layer (#4196)
add b5230a2756 HDDS-7923. [EC] Reconstruction is failing with IndexOutOfBoundsException (#4258)
add 975e835fc6 HDDS-7928. EC: Change ContainerReplicaPendingOps to store deadline rather than scheduled time (#4260)
add b3aac19f6b HDDS-7920. User cannot list their own volumes without access to root volume (#4254)
add a5aa549dc8 HDDS-7829. SCM to reject adding container to closed pipeline (#4212)
add 13a284834b HDDS-7944. Add dash-separated-options where missing (#3943)
add 7c2db17276 HDDS-7634. Recon: Show Datanode UUID on Pipeline page (#4135)
add a8d20de041 HDDS-7513. [Snapshot] Support list snapshots using fs -ls (#4134)
add 9b5b8d6b32 HDDS-7667. [Admin Doc] Observability: Add Grafana integration (#4267)
new 95d9f87f92 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
new c4a4cc0990 HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
new 28cfeab74b HDDS-7205. DiskBalancer CLI (#3739)
new 6ae2da8218 HDDS-7234. Add a common option for DiskBalancer commands (#3762)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (35791f59ba)
\
N -- N -- N refs/heads/HDDS-5713 (6ae2da8218)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 423 +++
.github/workflows/close-pending.yaml | 2 +-
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 405 +--
.github/workflows/scheduled_ci.yml | 22 +
LICENSE.txt | 10 +-
NOTICE.txt | 2 +-
README.md | 2 +-
dev-support/annotations/pom.xml | 114 -
.../annotations/ReplicateAnnotationProcessor.java | 77 -
.../services/javax.annotation.processing.Processor | 17 -
dev-support/ci/lib/_initialization.sh | 4 +-
hadoop-hdds/annotations/pom.xml | 34 +
.../annotations/ReplicateAnnotationProcessor.java | 80 +
.../RequestFeatureValidatorProcessor.java | 0
.../org/apache/ozone/annotations/package-info.java | 0
hadoop-hdds/client/pom.xml | 6 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 56 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 74 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 21 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 10 +
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +-
.../hdds/scm/storage/AbstractDataStreamOutput.java | 131 +
.../hdds/scm/storage/BlockDataStreamOutput.java | 733 ++++
.../hdds/scm/storage/BlockExtendedInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 64 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 75 +-
.../hdds/scm/storage/ByteBufferStreamOutput.java | 57 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 30 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 105 +-
.../hdds/scm/storage/MultipartInputStream.java | 246 ++
.../hadoop/hdds/scm/storage/PartInputStream.java | 38 +
.../hdds/scm/storage/RatisBlockOutputStream.java | 19 +-
.../hadoop/hdds/scm/storage/StreamBuffer.java | 59 +
.../hdds/scm/storage/StreamCommitWatcher.java | 208 ++
.../ozone/client/io/BadDataLocationException.java | 32 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 67 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 18 +-
.../client/io/ECBlockReconstructedInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hdds/scm/storage/DummyChunkInputStream.java | 2 +-
.../hdds/scm/storage/TestBlockInputStream.java | 135 +-
.../hdds/scm/storage/TestChunkInputStream.java | 9 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 8 +-
.../ozone/client/io/TestECBlockInputStream.java | 60 +-
.../TestECBlockReconstructedStripeInputStream.java | 20 +
hadoop-hdds/common/pom.xml | 18 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 45 +
.../org/apache/hadoop/hdds/client/BlockID.java | 5 +
.../hadoop/hdds/client/ContainerBlockID.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 94 +-
.../hadoop/hdds/client/RatisReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ReplicationConfig.java | 11 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 5 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 24 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 97 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 6 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 46 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 24 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 3 +
.../scm/container/common/helpers/ExcludeList.java | 4 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 2 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 2 +-
.../pipeline/DuplicatedPipelineIdException.java | 31 +
.../pipeline/InvalidPipelineStateException.java | 30 +
.../protocol/StorageContainerLocationProtocol.java | 9 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 140 +-
.../x509/certificate/utils/CertificateCodec.java | 5 +
.../org/apache/hadoop/hdds/server/JsonUtils.java | 49 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 17 +
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 4 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 92 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 8 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 4 +
.../ozone/container/common/helpers/ChunkInfo.java | 14 +
.../helpers/ContainerCommandRequestPBHelper.java | 1 +
.../java/org/apache/hadoop/util/CacheMetrics.java | 103 +
.../org/apache/hadoop/util/CheckedRunnable.java | 28 +
.../org/apache/hadoop/util/CheckedSupplier.java | 29 +
.../java/org/apache/hadoop/util/MetricUtil.java | 52 +
.../services/javax.annotation.processing.Processor | 2 +
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 358 +-
.../hadoop/ozone/audit/AuditLogTestUtils.java | 81 +
.../ozone/container/ContainerTestHelper.java | 30 +-
hadoop-hdds/config/pom.xml | 4 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 1 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-hdds/container-service/pom.xml | 23 +-
.../hdds/datanode/metadata/CRLDBDefinition.java | 5 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 143 +-
.../container/common/helpers/ContainerUtils.java | 49 +
.../container/common/impl/ContainerDataYaml.java | 8 +-
.../common/impl/ContainerLayoutVersion.java | 26 +-
.../ozone/container/common/impl/ContainerSet.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 66 +-
.../RandomContainerDeletionChoosingPolicy.java | 22 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 26 +-
.../common/interfaces/ContainerDispatcher.java | 10 +
.../common/interfaces/ContainerPacker.java | 3 +-
.../ozone/container/common/interfaces/Handler.java | 16 +
.../container/common/report/ReportPublisher.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 189 +-
.../common/statemachine/DatanodeQueueMetrics.java | 179 +
.../common/statemachine/DatanodeStateMachine.java | 94 +-
.../common/statemachine/EndpointStateMachine.java | 1 +
.../common/statemachine/StateContext.java | 59 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 64 +-
.../DeleteContainerCommandHandler.java | 74 +-
.../ReconstructECContainersCommandHandler.java | 25 +-
.../ReplicateContainerCommandHandler.java | 25 +-
.../states/endpoint/HeartbeatEndpointTask.java | 7 +
.../common/transport/server/XceiverServerGrpc.java | 4 +-
.../server/ratis/ContainerStateMachine.java | 79 +-
.../common/transport/server/ratis/LocalStream.java | 58 +
.../transport/server/ratis/XceiverServerRatis.java | 44 +-
.../container/common/utils/DatanodeStoreCache.java | 29 +-
.../container/common/utils/HddsVolumeUtil.java | 8 +-
.../common/utils/db/DatanodeDBProfile.java | 16 +-
.../ozone/container/common/volume/HddsVolume.java | 35 +-
.../container/common/volume/StorageVolume.java | 4 +
.../container/common/volume/VolumeInfoMetrics.java | 5 +
.../reconstruction/ECContainerOperationClient.java | 40 +-
.../ECReconstructionCommandInfo.java | 81 +-
.../ECReconstructionCoordinator.java | 77 +-
.../ECReconstructionCoordinatorTask.java | 86 +-
.../ec/reconstruction/ECReconstructionMetrics.java | 80 +
.../reconstruction/ECReconstructionSupervisor.java | 108 -
.../container/ec/reconstruction/TokenHelper.java | 16 +-
.../container/keyvalue/KeyValueContainer.java | 91 +-
.../container/keyvalue/KeyValueContainerCheck.java | 116 +-
.../container/keyvalue/KeyValueContainerData.java | 27 +-
.../KeyValueContainerMetadataInspector.java | 14 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 119 +-
.../container/keyvalue/TarContainerPacker.java | 171 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../helpers/KeyValueContainerLocationUtil.java | 41 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 10 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 14 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 16 +
.../keyvalue/impl/ChunkManagerFactory.java | 12 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +
.../keyvalue/impl/KeyValueStreamDataChannel.java | 277 ++
.../keyvalue/impl/StreamDataChannelBase.java | 96 +
.../keyvalue/interfaces/ChunkManager.java | 13 +
.../background/BlockDeletingService.java | 89 +-
.../StaleRecoveringContainerScrubbingService.java | 5 +-
.../container/metadata/AbstractDatanodeStore.java | 42 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 24 +-
.../ozone/container/metadata/DatanodeStore.java | 3 +
.../metadata/DatanodeStoreSchemaThreeImpl.java | 90 +
.../ozoneimpl/AbstractContainerScanner.java | 136 +
.../ozoneimpl/AbstractContainerScannerMetrics.java | 90 +
.../container/ozoneimpl/ContainerController.java | 3 +-
.../container/ozoneimpl/ContainerDataScanner.java | 135 +-
.../ozoneimpl/ContainerDataScannerMetrics.java | 69 +
.../ozoneimpl/ContainerDataScrubberMetrics.java | 120 -
.../ozoneimpl/ContainerMetadataScanner.java | 112 +-
.../ozoneimpl/ContainerMetadataScannerMetrics.java | 44 +
.../ContainerMetadataScrubberMetrics.java | 98 -
.../ozoneimpl/ContainerScannerConfiguration.java | 165 +
.../ozoneimpl/ContainerScrubberConfiguration.java | 140 -
.../ozoneimpl/OnDemandContainerScanner.java | 178 +
.../ozoneimpl/OnDemandScannerMetrics.java | 42 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 71 +-
.../replication/AbstractReplicationTask.java | 123 +
.../container/replication/ContainerDownloader.java | 3 +-
.../container/replication/ContainerImporter.java | 127 +
.../replication/ContainerReplicationSource.java | 6 +-
.../container/replication/ContainerUploader.java | 33 +
.../replication/CopyContainerCompression.java | 129 +
.../replication/CopyContainerResponseStream.java | 47 +
.../replication/DownloadAndImportReplicator.java | 113 +-
.../replication/GrpcContainerUploader.java | 99 +
.../container/replication/GrpcOutputStream.java | 41 +-
.../replication/GrpcReplicationClient.java | 37 +-
.../replication/GrpcReplicationService.java | 28 +-
.../container/replication/MeasuredReplicator.java | 23 +-
.../OnDemandContainerReplicationSource.java | 11 +-
.../container/replication/PushReplicator.java | 82 +
.../container/replication/ReplicationServer.java | 31 +-
.../replication/ReplicationSupervisor.java | 212 +-
.../replication/ReplicationSupervisorMetrics.java | 10 +-
.../container/replication/ReplicationTask.java | 80 +-
.../replication/SendContainerOutputStream.java | 48 +
.../replication/SendContainerRequestHandler.java | 133 +
.../replication/SimpleContainerDownloader.java | 34 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 3 +-
.../protocol/commands/CloseContainerCommand.java | 10 +
.../protocol/commands/DeleteContainerCommand.java | 16 +
.../commands/ReconstructECContainersCommand.java | 15 +
.../commands/ReplicateContainerCommand.java | 84 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 35 +
.../webapps/hddsDatanode/dn-overview.html | 2 +
.../metadata/TestDatanodeCRLStoreImpl.java | 24 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 16 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 317 +-
.../ozone/container/common/ContainerTestUtils.java | 39 +-
.../container/common/TestBlockDeletingService.java | 219 +-
.../common/TestContainerLayoutVersion.java | 4 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../container/common/TestDatanodeStateMachine.java | 60 +-
.../container/common/TestDatanodeStoreCache.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 29 +-
.../TestSchemaTwoBackwardsCompatibility.java | 4 +-
...stStaleRecoveringContainerScrubbingService.java | 38 +-
.../common/helpers/TestContainerUtils.java | 14 +-
.../container/common/impl/TestHddsDispatcher.java | 68 +
.../container/common/report/TestReportManager.java | 2 +-
.../common/report/TestReportPublisher.java | 39 +-
.../statemachine/TestDatanodeConfiguration.java | 4 +-
.../common/statemachine/TestStateContext.java | 102 +-
.../TestDeleteContainerCommandHandler.java | 135 +
.../states/datanode/TestRunningDatanodeState.java | 8 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 103 +-
.../container/common/utils/TestHddsVolumeUtil.java | 8 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 31 +-
.../container/common/volume/TestHddsVolume.java | 10 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 35 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 2 +-
.../TestECReconstructionSupervisor.java | 71 -
.../container/keyvalue/TestKeyValueContainer.java | 330 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueContainerIntegrityChecks.java | 4 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +
.../TestKeyValueContainerMetadataInspector.java | 8 +-
.../container/keyvalue/TestKeyValueHandler.java | 10 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 15 +-
.../container/keyvalue/TestTarContainerPacker.java | 115 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 20 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 25 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 10 +-
.../keyvalue/impl/TestChunkManagerDummyImpl.java | 4 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 6 +-
.../keyvalue/impl/TestFilePerChunkStrategy.java | 8 +-
.../impl/TestKeyValueStreamDataChannel.java | 313 ++
.../container/ozoneimpl/TestContainerReader.java | 13 +-
.../TestContainerScannerConfiguration.java | 109 +
.../ozoneimpl/TestContainerScannerMetrics.java | 199 ++
.../TestContainerScrubberConfiguration.java | 97 -
.../ozoneimpl/TestContainerScrubberMetrics.java | 160 -
.../ozoneimpl/TestOnDemandContainerScanner.java | 157 +
.../container/ozoneimpl/TestOzoneContainer.java | 7 +-
.../replication/GrpcOutputStreamTest.java | 227 ++
.../ReplicationSupervisorScheduling.java | 99 +-
.../replication/TestCopyContainerCompression.java | 95 +
.../TestCopyContainerResponseStream.java | 50 +
.../replication/TestGrpcOutputStream.java | 213 --
.../replication/TestMeasuredReplicator.java | 68 +-
.../container/replication/TestPushReplicator.java | 164 +
.../replication/TestReplicationConfig.java | 4 +-
.../replication/TestReplicationSupervisor.java | 331 +-
.../replication/TestSendContainerOutputStream.java | 75 +
.../replication/TestSimpleContainerDownloader.java | 71 +-
.../stream/TestDirstreamClientHandler.java | 34 +-
.../container/stream/TestStreamingServer.java | 18 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 4 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 23 +-
.../TestReconstructionECContainersCommands.java | 27 +-
.../dev-support/checkstyle/suppressions.xml | 1 +
hadoop-hdds/docs/content/concept/Recon.md | 1 +
hadoop-hdds/docs/content/design/trash.md | 5 -
.../content/feature/GrafanaOzoneObjectMetrics.png | Bin 0 -> 319286 bytes
.../docs/content/feature/GrafanaOzoneOverall.png | Bin 0 -> 507556 bytes
.../content/feature/GrafanaOzoneRPCMetrics.png | Bin 0 -> 447935 bytes
.../docs/content/feature/Nonrolling-Upgrade.md | 6 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Observability.md | 42 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 4 +-
.../docs/content/feature/Reconfigurability.md | 65 +
.../docs/content/feature/Reconfigurability.zh.md | 62 +
hadoop-hdds/docs/content/feature/SCM-HA.md | 7 +
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 73 +-
.../content/feature/Streaming-Write-Pipeline.md | 136 +
hadoop-hdds/docs/content/interface/Cli.md | 62 +
hadoop-hdds/docs/content/interface/Ofs.md | 32 +-
hadoop-hdds/docs/content/interface/S3.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 2 +-
hadoop-hdds/docs/content/security/SecurityAcls.md | 9 +-
.../docs/content/security/SecurityAcls.zh.md | 7 +-
.../docs/content/security/SecurityWithRanger.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
hadoop-hdds/docs/content/tools/LogsInOzone.md | 45 +
hadoop-hdds/docs/pom.xml | 4 +-
.../themes/ozonedoc/layouts/partials/header.html | 22 +
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../io/erasurecode/rawcoder/package-info.java | 29 -
.../framework/dev-support/findbugsExcludeFile.xml | 4 -
hadoop-hdds/framework/pom.xml | 43 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 35 +
.../hadoop/hdds/protocol/ReconfigureProtocol.java | 56 +
.../ReconfigureProtocolClientSideTranslatorPB.java | 178 +
.../hdds/protocolPB/ReconfigureProtocolPB.java | 36 +
.../ReconfigureProtocolServerSideTranslatorPB.java | 128 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 21 +-
.../hadoop/hdds/security/OzoneSecretKey.java | 9 +-
.../hadoop/hdds/security/OzoneSecretManager.java | 54 +-
.../hadoop/hdds/security/ssl/KeyStoresFactory.java | 69 +
.../hdds/security/ssl/MonitoringTimerTask.java | 77 +
.../security/ssl/PemFileBasedKeyStoresFactory.java | 199 ++
.../hdds/security/ssl/ReloadingX509KeyManager.java | 178 +
.../security/ssl/ReloadingX509TrustManager.java | 152 +
.../hadoop/hdds/security/ssl/package-info.java | 22 +
.../token/ContainerTokenSecretManager.java | 5 +-
.../token/OzoneBlockTokenSecretManager.java | 5 +-
.../token/ShortLivedTokenSecretManager.java | 9 +-
.../x509/certificate/authority/BaseApprover.java | 4 +-
.../certificate/authority/DefaultApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 126 +-
.../authority/PKIProfiles/DefaultCAProfile.java | 84 -
.../authority/PKIProfiles/DefaultProfile.java | 346 --
.../authority/PKIProfiles/PKIProfile.java | 149 -
.../authority/PKIProfiles/package-info.java | 33 -
.../authority/profile/DefaultCAProfile.java | 84 +
.../authority/profile/DefaultProfile.java | 346 ++
.../certificate/authority/profile/PKIProfile.java | 149 +
.../authority/profile/package-info.java | 33 +
.../x509/certificate/client/CertificateClient.java | 65 +-
.../client/CertificateNotification.java | 35 +
.../client/CommonCertificateClient.java | 20 +-
.../certificate/client/DNCertificateClient.java | 101 +-
.../client/DefaultCertificateClient.java | 684 +++-
.../certificate/client/OMCertificateClient.java | 57 -
.../certificate/client/ReconCertificateClient.java | 104 +-
.../certificate/client/SCMCertificateClient.java | 28 +-
.../certificate/utils/CertificateSignRequest.java | 344 ++
.../certificate/utils/SelfSignedCertificate.java | 306 ++
.../certificates/utils/CertificateSignRequest.java | 344 --
.../certificates/utils/SelfSignedCertificate.java | 302 --
.../x509/certificates/utils/package-info.java | 22 -
.../x509/exception/CertificateException.java | 95 +
.../hdds/security/x509/exception/package-info.java | 23 +
.../x509/exceptions/CertificateException.java | 89 -
.../security/x509/exceptions/package-info.java | 23 -
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 16 +-
.../hdds/security/x509/keys/SecurityUtil.java | 36 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 92 +
.../hadoop/hdds/server/ServiceRuntimeInfoImpl.java | 24 +-
.../hadoop/hdds/server/events/EventExecutor.java | 21 +
.../FixedThreadPoolWithAffinityExecutor.java | 219 +-
.../hadoop/hdds/server/events/IEventInfo.java | 28 +
.../hdds/server/events/SingleThreadExecutor.java | 6 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 10 +-
.../hadoop/hdds/server/http/HttpServer2.java | 3 +
.../hdds/server/http/HttpServer2Metrics.java | 94 +
.../hdds/server/http/PrometheusMetricsSink.java | 109 +-
.../hdds/server/http/RatisDropwizardExports.java | 86 +-
.../hadoop/hdds/utils/BooleanTriFunction.java | 37 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 80 +-
.../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 120 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 18 +-
.../org/apache/hadoop/hdds/utils/MetricsUtil.java | 100 +
.../hdds/utils/PrometheusMetricsSinkUtil.java | 116 +
.../hadoop/hdds/utils/TableCacheMetrics.java | 92 +
.../apache/hadoop/hdds/utils/UgiMetricsUtil.java | 68 +
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 25 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 6 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 64 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 67 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 98 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 6 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 403 ++-
.../org/apache/hadoop/hdds/utils/db/Table.java | 12 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 4 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 8 +-
.../hadoop/hdds/utils/db/cache/CacheStats.java | 45 +
.../hdds/utils/db/cache/CacheStatsRecorder.java | 58 +
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 16 +-
.../hdds/utils/db/cache/PartialTableCache.java | 13 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 6 +
.../db/managed/ManagedColumnFamilyOptions.java | 13 +
.../db/managed/ManagedCompactRangeOptions.java | 33 +
.../hdds/utils/db/managed/ManagedObject.java | 22 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 25 +-
.../hadoop/hdds/utils/db/managed/ManagedSlice.java | 42 +
.../ssl/TestPemFileBasedKeyStoresFactory.java | 255 ++
.../security/ssl/TestReloadingX509KeyManager.java | 83 +
.../ssl/TestReloadingX509TrustManager.java | 83 +
.../token/TestOzoneBlockTokenSecretManager.java | 6 +-
.../hdds/security/token/TokenVerifierTests.java | 2 +-
.../hdds/security/x509/CertificateClientTest.java | 273 ++
.../x509/certificate/authority/MockApprover.java | 2 +-
.../certificate/authority/TestDefaultCAServer.java | 105 +-
.../certificate/authority/TestDefaultProfile.java | 4 +-
.../client/TestCertificateClientInit.java | 217 --
.../client/TestDefaultCertificateClient.java | 318 +-
.../client/TestDnCertificateClientInit.java | 158 +
.../x509/certificate/utils/TestCRLCodec.java | 10 +-
.../certificate/utils/TestCertificateCodec.java | 28 +-
.../utils/TestCertificateSignRequest.java | 297 ++
.../certificate/utils/TestRootCertificate.java | 288 ++
.../certificates/TestCertificateSignRequest.java | 298 --
.../x509/certificates/TestRootCertificate.java | 290 --
.../security/x509/certificates/package-info.java | 22 -
.../hadoop/hdds/server/events/TestEventQueue.java | 24 +-
.../server/http/TestHttpServer2MetricsTest.java | 94 +
.../http/TestPrometheusMetricsIntegration.java | 259 ++
.../server/http/TestPrometheusMetricsSink.java | 221 --
.../hdds/utils/TestDecayRpcSchedulerUtil.java | 108 +
.../hdds/utils/TestPrometheusMetricsSinkUtil.java | 229 ++
.../hadoop/hdds/utils/TestUgiMetricsUtil.java | 63 +
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 16 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 44 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 22 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 36 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 5 +-
.../interface-admin/src/main/resources/proto.lock | 679 +++-
hadoop-hdds/interface-client/pom.xml | 6 +-
.../src/main/proto/DatanodeClientProtocol.proto | 27 +-
.../src/main/proto/ReconfigureProtocol.proto | 80 +
.../interface-client/src/main/proto/hdds.proto | 1 +
.../interface-client/src/main/resources/proto.lock | 991 +++++-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 11 +
.../src/main/proto/ScmServerProtocol.proto | 2 +
.../interface-server/src/main/resources/proto.lock | 840 +++--
hadoop-hdds/pom.xml | 18 +-
hadoop-hdds/rocksdb-checkpoint-differ/README.md | 18 +
.../dev-support/findbugsExcludeFile.xml | 0
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 193 +
.../ozone/rocksdb/util/ManagedSstFileReader.java | 128 +
.../org/apache/ozone/rocksdb/util/RdbUtil.java | 61 +
.../apache/ozone/rocksdb/util/package-info.java | 22 +
.../org/apache/ozone/rocksdiff/CompactionNode.java | 80 +
.../apache/ozone/rocksdiff/DifferSnapshotInfo.java | 63 +
.../apache/ozone/rocksdiff/RelationshipEdge.java | 30 +
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 1377 ++++++++
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 54 +
.../org/apache/ozone/rocksdiff}/package-info.java | 0
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 1221 +++++++
.../src/test/resources/log4j.properties | 0
hadoop-hdds/server-scm/pom.xml | 14 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 254 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 34 +
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 34 +-
.../scm/block/DeletedBlockLogStateManager.java | 3 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 8 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 11 +-
.../scm/container/CloseContainerEventHandler.java | 5 +-
.../hdds/scm/container/ContainerManager.java | 32 +-
.../hdds/scm/container/ContainerManagerImpl.java | 48 +-
.../hdds/scm/container/ContainerReplica.java | 2 +-
.../hdds/scm/container/ContainerReportHandler.java | 13 +-
.../scm/container/ContainerStateManagerImpl.java | 174 +-
.../IncrementalContainerReportHandler.java | 9 +-
.../scm/container/RatisContainerReplicaCount.java | 273 --
.../balancer/AbstractFindTargetGreedy.java | 54 +-
.../scm/container/balancer/ContainerBalancer.java | 1125 +-----
.../balancer/ContainerBalancerMetrics.java | 29 +
.../ContainerBalancerSelectionCriteria.java | 22 +-
.../container/balancer/ContainerBalancerTask.java | 1079 ++++++
.../scm/container/balancer/FindSourceGreedy.java | 55 +-
.../scm/container/balancer/FindSourceStrategy.java | 11 +
.../FindTargetGreedyByNetworkTopology.java | 21 +
.../balancer/FindTargetGreedyByUsageInfo.java | 23 +
.../scm/container/balancer/FindTargetStrategy.java | 9 +
.../ContainerPlacementStatusDefault.java | 36 +-
.../algorithms/SCMContainerPlacementCapacity.java | 12 +-
.../algorithms/SCMContainerPlacementRackAware.java | 22 +-
.../SCMContainerPlacementRackScatter.java | 310 +-
.../algorithms/SCMContainerPlacementRandom.java | 16 +-
.../AbstractOverReplicationHandler.java | 32 +-
.../replication/ContainerCheckRequest.java | 122 +
.../replication/ContainerHealthCheck.java | 35 -
.../replication/ContainerHealthResult.java | 102 +-
.../replication/ContainerReplicaCount.java | 7 +-
.../container/replication/ContainerReplicaOp.java | 8 +-
.../replication/ContainerReplicaPendingOps.java | 130 +-
.../ContainerReplicaPendingOpsSubscriber.java | 39 +
.../DatanodeCommandCountUpdatedHandler.java | 48 +
.../replication/ECContainerHealthCheck.java | 86 -
.../replication/ECContainerReplicaCount.java | 146 +-
.../replication/ECMisReplicationHandler.java | 67 +
.../replication/ECOverReplicationHandler.java | 163 +-
.../replication/ECUnderReplicationHandler.java | 370 +-
.../LegacyRatisContainerReplicaCount.java | 51 +
.../replication/LegacyReplicationManager.java | 906 +++--
.../replication/MisReplicationHandler.java | 200 ++
.../replication/OverReplicatedProcessor.java | 96 +-
.../replication/RatisContainerReplicaCount.java | 496 +++
.../replication/RatisMisReplicationHandler.java | 76 +
.../replication/RatisOverReplicationHandler.java | 263 ++
.../replication/RatisUnderReplicationHandler.java | 250 ++
.../container/replication/ReplicationManager.java | 576 ++-
.../replication/ReplicationManagerMetrics.java | 107 +-
.../container/replication/ReplicationQueue.java | 73 +
.../replication/UnderReplicatedProcessor.java | 107 +-
.../replication/UnhealthyReplicationHandler.java | 5 +-
.../replication/UnhealthyReplicationProcessor.java | 152 +
.../replication/health/AbstractCheck.java | 71 +
.../health/ClosedWithUnhealthyReplicasHandler.java | 129 +
.../health/ClosingContainerHandler.java | 72 +
.../health/DeletingContainerHandler.java | 100 +
.../health/ECReplicationCheckHandler.java | 211 ++
.../replication/health/EmptyContainerHandler.java | 126 +
.../container/replication/health/HealthCheck.java | 56 +
.../health/MismatchedReplicasHandler.java | 105 +
.../replication/health/OpenContainerHandler.java | 90 +
.../health/QuasiClosedContainerHandler.java | 135 +
.../health/RatisReplicationCheckHandler.java | 233 ++
.../container/replication/health/package-info.java | 23 +
.../container/report/ContainerReportValidator.java | 104 +
.../hdds/scm/container/report/package-info.java | 19 +
.../scm/container/states/ContainerAttribute.java | 14 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 13 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 8 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 8 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 24 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 54 +-
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 105 +
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 66 +-
.../hdds/scm/node/CommandQueueReportHandler.java | 3 +-
.../hadoop/hdds/scm/node/DatanodeAdminMonitor.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 70 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 29 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 12 +
.../hdds/scm/node/NodeDecommissionManager.java | 8 +-
.../hdds/scm/node/NodeDecommissionMetrics.java | 290 ++
.../apache/hadoop/hdds/scm/node/NodeManager.java | 19 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 51 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 42 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 47 +-
.../pipeline/PipelinePlacementPolicyFactory.java | 52 +
.../scm/pipeline/PipelineStateManagerImpl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 23 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 34 +-
.../scm/pipeline/WritableECContainerProvider.java | 11 +-
...inerLocationProtocolServerSideTranslatorPB.java | 17 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 4 -
.../hdds/scm/server/ContainerReportQueue.java | 378 ++
.../hdds/scm/server/SCMBlockProtocolServer.java | 14 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 34 +-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 1 +
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 86 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 58 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 6 -
.../hdds/scm/server/StorageContainerManager.java | 357 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 102 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 431 ++-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 201 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 16 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 15 +-
.../scm/container/TestContainerManagerImpl.java | 12 +-
.../scm/container/TestContainerReportHandler.java | 59 +
.../TestIncrementalContainerReportHandler.java | 148 +-
.../container/balancer/TestContainerBalancer.java | 954 +----
.../balancer/TestContainerBalancerTask.java | 1039 ++++++
.../container/balancer/TestFindTargetStrategy.java | 34 +
.../algorithms/TestContainerPlacementFactory.java | 24 +-
.../TestContainerPlacementStatusDefault.java | 22 +-
.../TestSCMContainerPlacementRackScatter.java | 170 +-
.../container/replication/ReplicationTestUtil.java | 99 +-
.../TestContainerReplicaPendingOps.java | 199 +-
.../TestDatanodeCommandCountUpdatedHandler.java | 49 +
.../replication/TestECContainerHealthCheck.java | 221 --
.../replication/TestECContainerReplicaCount.java | 191 +-
.../replication/TestECMisReplicationHandler.java | 175 +
.../replication/TestECOverReplicationHandler.java | 145 +-
.../replication/TestECUnderReplicationHandler.java | 485 ++-
.../replication/TestLegacyReplicationManager.java | 3677 +++++++++++---------
.../replication/TestMisReplicationHandler.java | 184 +
.../replication/TestOverReplicatedProcessor.java | 53 +-
.../TestRatisContainerReplicaCount.java | 104 +-
.../TestRatisMisReplicationHandler.java | 179 +
.../TestRatisOverReplicationHandler.java | 285 ++
.../TestRatisUnderReplicationHandler.java | 234 ++
.../replication/TestReplicationManager.java | 475 ++-
.../replication/TestReplicationManagerMetrics.java | 2 +
.../replication/TestUnderReplicatedProcessor.java | 85 +-
.../TestClosedWithUnhealthyReplicasHandler.java | 194 ++
.../health/TestClosingContainerHandler.java | 223 ++
.../health/TestDeletingContainerHandler.java | 238 ++
.../health/TestECReplicationCheckHandler.java | 732 ++++
.../health/TestEmptyContainerHandler.java | 229 ++
.../health/TestMismatchedReplicasHandler.java | 273 ++
.../health/TestOpenContainerHandler.java | 168 +
.../health/TestQuasiClosedContainerHandler.java | 238 ++
.../health/TestRatisReplicationCheckHandler.java | 676 ++++
.../container/replication/health/package-info.java | 24 +
.../report/TestContainerReportValidator.java | 77 +
.../hdds/scm/container/report/package-info.java | 22 +
.../scm/metadata/TestSCMMetadataStoreImpl.java | 67 +
.../scm/node/DatanodeAdminMonitorTestUtil.java | 209 ++
.../hadoop/hdds/scm/node/TestCommandQueue.java | 101 +
.../scm/node/TestCommandQueueReportHandler.java | 30 +-
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 224 +-
.../hdds/scm/node/TestNodeDecommissionMetrics.java | 343 ++
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 74 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 65 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 215 ++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 15 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 22 +-
.../pipeline/TestWritableECContainerProvider.java | 5 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 27 +-
.../ozone/container/common/TestEndPoint.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 15 +-
hadoop-hdds/test-utils/pom.xml | 14 +-
.../org/apache/ozone/test/GenericTestUtils.java | 93 +
.../java/org/apache/ozone/test/SpyInputStream.java | 47 +
.../org/apache/ozone/test/SpyOutputStream.java | 47 +
hadoop-hdds/tools/pom.xml | 14 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 12 +-
.../hdds/scm/cli/ContainerOperationClient.java | 169 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 1 +
.../hadoop/hdds/scm/cli/cert/CleanExpired.java | 117 +
.../hdds/scm/cli/container/InfoSubcommand.java | 6 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 12 +-
.../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 100 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 22 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 4 +
hadoop-ozone/client/pom.xml | 10 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 56 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 72 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 40 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 11 +-
.../apache/hadoop/ozone/client/OzoneSnapshot.java | 145 +
.../client/checksum/BaseFileChecksumHelper.java | 12 +-
.../client/checksum/ChecksumHelperFactory.java | 52 +
.../client/checksum/ECBlockChecksumComputer.java | 221 ++
.../client/checksum/ECFileChecksumHelper.java | 208 ++
.../client/io/BlockDataStreamOutputEntry.java | 287 ++
.../client/io/BlockDataStreamOutputEntryPool.java | 290 ++
.../ozone/client/io/BlockOutputStreamEntry.java | 13 +
.../client/io/BlockOutputStreamEntryPool.java | 23 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 34 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 4 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 246 +-
.../ozone/client/io/KeyDataStreamOutput.java | 510 +++
.../hadoop/ozone/client/io/KeyInputStream.java | 436 +--
.../hadoop/ozone/client/io/KeyOutputStream.java | 25 +-
.../client/io/MultipartCryptoKeyInputStream.java | 382 --
.../ozone/client/io/OzoneCryptoInputStream.java | 157 +-
.../ozone/client/io/OzoneDataStreamOutput.java | 70 +
.../hadoop/ozone/client/io/OzoneOutputStream.java | 25 +-
.../ozone/client/protocol/ClientProtocol.java | 106 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 446 ++-
.../hadoop/ozone/client/MockOmTransport.java | 36 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 62 +-
hadoop-ozone/common/pom.xml | 18 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 38 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 77 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 7 +-
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 123 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 32 +-
.../java/org/apache/hadoop/ozone/om/S3Batcher.java | 46 +
.../org/apache/hadoop/ozone/om/S3SecretCache.java | 39 +
.../apache/hadoop/ozone/om/S3SecretFunction.java | 28 +
.../hadoop/ozone/om/S3SecretLockedManager.java | 100 +
.../apache/hadoop/ozone/om/S3SecretManager.java | 81 +-
.../org/apache/hadoop/ozone/om/S3SecretStore.java | 56 +
.../hadoop/ozone/om/exceptions/OMException.java | 6 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 13 +
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 110 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 72 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 18 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 55 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 109 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 19 +
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 515 +++
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 98 +
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 167 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 36 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 12 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 139 +-
.../ozone/om/protocolPB/OMAdminProtocolPB.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 145 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 -
.../ozone/security/OzoneTokenIdentifier.java | 3 +-
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 233 ++
.../apache/hadoop/ozone/snapshot/package-info.java | 22 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +
.../main/resources/ozone-version-info.properties | 1 +
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 91 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 11 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 6 +-
.../ozone/om/helpers/TestOmSnapshotInfo.java | 147 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 9 +-
hadoop-ozone/csi/pom.xml | 13 +-
hadoop-ozone/datanode/pom.xml | 4 +-
.../dev-support/checks/_mvn_unit_report.sh | 2 +-
hadoop-ozone/dev-support/checks/coverage.sh | 6 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
hadoop-ozone/dev-support/intellij/log4j.properties | 2 +
.../dev-support/intellij/ozone-site-ha.xml | 174 +
hadoop-ozone/dev-support/intellij/ozone-site.xml | 20 +
.../intellij/runConfigurations/Datanode1-ha.xml | 33 +
.../intellij/runConfigurations/Datanode2-ha.xml | 33 +
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3-ha.xml | 33 +
.../intellij/runConfigurations/Datanode3.xml | 2 +-
.../intellij/runConfigurations/OzoneFsShell-ha.xml | 36 +
.../intellij/runConfigurations/OzoneManager-ha.xml | 33 +
.../runConfigurations/OzoneManagerInit-ha.xml | 33 +
.../intellij/runConfigurations/OzoneShell-ha.xml | 33 +
.../runConfigurations/PrimordialSCM-ha.xml | 33 +
.../runConfigurations/PrimordialSCMInit-ha.xml | 33 +
.../intellij/runConfigurations/Recon-ha.xml | 33 +
.../intellij/runConfigurations/Scm2-ha.xml | 33 +
.../runConfigurations/Scm2Bootstrap-ha.xml | 33 +
.../intellij/runConfigurations/Scm3-ha.xml | 33 +
.../runConfigurations/Scm3Bootstrap-ha.xml | 33 +
.../intellij/runConfigurations/ScmRoles.xml | 33 +
.../runConfigurations/StorageContainerManager.xml | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 -
hadoop-ozone/dist/pom.xml | 16 +-
.../{ozone-csi => ozone-legacy-bucket}/.env | 0
.../src/main/compose/ozone-legacy-bucket/README.md | 21 +
.../docker-compose.yaml | 0
.../main/compose/ozone-legacy-bucket/docker-config | 52 +
.../src/main/compose/ozone-legacy-bucket/test.sh | 35 +
.../main/compose/ozone-om-ha/docker-compose.yaml | 5 +
.../dist/src/main/compose/ozone/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../src/main/compose/ozonesecure/docker-config | 9 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 153 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 4 +-
.../main/license/bin/licenses/LICENSE-angular.txt | 2 +-
.../src/main/license/bin/licenses/LICENSE-d3.txt | 2 +-
.../license/bin/licenses/LICENSE-glyphicons.txt | 21 +
.../main/license/bin/licenses/LICENSE-guava.txt | 13 +
.../LICENSE-javax.activation-activation.txt | 3 -
.../LICENSE-net.sf.jopt-simple-jopt-simple.txt | 24 -
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 -
.../license/bin/licenses/LICENSE-org.aspectj.txt | 279 ++
.../bin/licenses/LICENSE-org.ow2.asm-asm.txt | 1 -
.../license/bin/licenses/LICENSE-org.slf4j.txt | 2 +-
.../bin/licenses/NOTICE-ratis-thirdparty-misc.txt | 318 ++
.../bin/licenses/NOTICE-ratis-thirtparty-misc.txt | 340 --
hadoop-ozone/dist/src/main/license/jar-report.txt | 37 +-
.../main/license/src/licenses/LICENSE-angular.txt | 2 +-
.../license/src/licenses/LICENSE-bootstrap.txt | 21 +
.../src/main/license/src/licenses/LICENSE-d3.txt | 4 +-
.../license/src/licenses/LICENSE-glyphicons.txt | 21 +
.../main/license/src/licenses/LICENSE-guava.txt | 13 +
.../main/license/src/licenses/LICENSE-jquery.txt | 2 +-
.../src/main/license/src/licenses/LICENSE-nvd3.txt | 17 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/pipeline.robot | 5 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 7 +-
.../main/smoketest/basic/ozone-shell-single.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +-
.../smoketest/compatibility/dn-one-rocksdb.robot | 2 +-
.../dist/src/main/smoketest/createbucketenv.robot | 2 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 2 +-
.../dist/src/main/smoketest/ec/basic.robot | 6 +-
.../dist/src/main/smoketest/ec/ozonefs.robot | 61 +
.../src/main/smoketest/freon/read-write-key.robot | 53 +
.../dist/src/main/smoketest/gdpr/gdpr.robot | 8 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 37 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 12 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 8 +-
.../main/smoketest/recon/recon-fso-nssummary.robot | 146 -
.../src/main/smoketest/recon/recon-nssummary.robot | 148 +
.../src/main/smoketest/s3/MultipartUpload.robot | 1 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 5 +
.../dist/src/main/smoketest/s3/objectputget.robot | 26 +
.../main/smoketest/security/ozone-secure-fs.robot | 51 +
.../smoketest/security/ozone-secure-tenant.robot | 7 +
.../src/main/smoketest/snapshot/snapshot-fs.robot | 60 +
.../main/smoketest/snapshot/snapshot-setup.robot | 63 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 57 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 18 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 18 +-
.../src/shell/conf/ozone-shell-log4j.properties | 34 -
.../src/shell/conf/s3g-audit-log4j2.properties | 18 +-
.../src/shell/conf/scm-audit-log4j2.properties | 18 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 24 +-
.../dist/src/shell/ozone/ozone-functions.sh | 87 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 -
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 103 +-
.../fs/contract/AbstractContractUnbufferTest.java | 160 -
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 111 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 221 ++
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 79 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 177 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 385 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 103 +
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 158 +
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 344 ++
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 544 ++-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 36 +-
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 52 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 33 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 3 +
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 32 +-
.../TestContainerStateManagerIntegration.java | 2 +-
.../container/TestScmApplyTransactionFailure.java | 147 +
.../metrics/TestSCMContainerManagerMetrics.java | 81 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 42 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 40 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 22 +-
.../TestRatisPipelineCreateAndDestroy.java | 12 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 33 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 35 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 52 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 20 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../hadoop/ozone/TestContainerOperations.java | 34 +
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 21 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 443 ++-
.../hadoop/ozone/TestStorageContainerManager.java | 157 +-
.../ozone/TestStorageContainerManagerHelper.java | 22 +-
.../ozone/client/CertificateClientTestImpl.java | 262 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 3 -
.../client/rpc/TestBlockDataStreamOutput.java | 273 ++
.../ozone/client/rpc/TestBlockOutputStream.java | 26 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 26 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 2 -
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 -
.../rpc/TestCloseContainerHandlingByClient.java | 9 -
.../client/rpc/TestContainerStateMachine.java | 2 -
.../TestContainerStateMachineFailureOnRead.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 1 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../rpc/TestContainerStateMachineStream.java | 219 ++
.../client/rpc/TestDeleteWithSlowFollower.java | 3 -
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 29 +-
.../client/rpc/TestFailureHandlingByClient.java | 11 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 1 -
.../rpc/TestMultiBlockWritesWithDnFailures.java | 4 -
.../client/rpc/TestOzoneAtRestEncryption.java | 7 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 441 +--
...estOzoneClientRetriesOnExceptionFlushDelay.java | 4 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 3 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 351 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 132 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 81 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 4 +-
.../client/rpc/read/TestChunkInputStream.java | 6 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 -
.../ozone/client/rpc/read/TestKeyInputStream.java | 6 +-
.../ozone/container/TestECContainerRecovery.java | 134 +-
.../apache/hadoop/ozone/container/TestHelper.java | 42 +-
.../commandhandler/TestBlockDeletion.java | 25 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../commandhandler/TestCloseContainerHandler.java | 1 -
.../commandhandler/TestDeleteContainerHandler.java | 1 -
.../metrics/TestDatanodeQueueMetrics.java | 111 +
.../ozoneimpl/TestOzoneContainerWithTLS.java | 239 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 4 +-
.../replication/TestContainerReplication.java | 173 +
.../ozone/container/replication/package-info.java | 22 +
.../container/server/TestContainerServer.java | 3 +-
.../server/TestSecureContainerServer.java | 10 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 1 -
.../hadoop/ozone/dn/scanner/TestDataScanner.java | 240 ++
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 233 --
.../TestDatanodeHddsVolumeFailureDetection.java | 36 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 269 ++
.../hadoop/ozone/fsck/TestContainerMapper.java | 8 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 +
.../ozone/om/TestContainerReportWithKeys.java | 1 -
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 491 ++-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 1 +
.../org/apache/hadoop/ozone/om/TestLDBCli.java | 326 ++
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 65 +-
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 24 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 4 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 52 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 174 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 2 -
.../ozone/om/TestOmContainerLocationCache.java | 674 ++++
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 187 -
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 137 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 662 ++++
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 581 ++++
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 11 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 62 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 4 +
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 3 -
.../hadoop/ozone/om/TestSecureOzoneManager.java | 20 +-
.../ozone/om/multitenant/RangerUserRequest.java | 3 +-
.../om/multitenant/TestRangerBGSyncService.java | 707 ----
.../ozone/om/service/TestRangerBGSyncService.java | 712 ++++
.../om/snapshot/TestOzoneSnapshotRestore.java | 333 ++
.../ozone/parser/TestOzoneHARatisLogParser.java | 61 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 4 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 40 +-
.../ozone/recon/TestReconWithOzoneManager.java | 57 -
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 14 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../hadoop/ozone/reconfig/TestOmReconfigure.java | 100 +
.../hadoop/ozone/scm/TestAllocateContainer.java | 39 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 26 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 84 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 47 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 5 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 51 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 30 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 22 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 103 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 208 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 29 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 35 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 30 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 69 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 98 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 2 +-
.../src/test/resources/contract/ozone.xml | 10 +
.../src/test/resources/testSequenceFile | Bin 0 -> 96 bytes
hadoop-ozone/interface-client/pom.xml | 6 +-
.../src/main/proto/OmClientProtocol.proto | 131 +
.../interface-client/src/main/resources/proto.lock | 2672 +++++++++++---
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 37 +-
.../ozone/om/codec/OmDBSnapshotInfoCodec.java | 58 +
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 45 +-
hadoop-ozone/ozone-manager/pom.xml | 29 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 214 +-
.../hadoop/ozone/om/DirectoryDeletingService.java | 299 --
.../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 31 -
.../apache/hadoop/ozone/om/KeyDeletingService.java | 313 --
.../org/apache/hadoop/ozone/om/KeyManager.java | 17 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 1180 ++-----
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +-
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 149 +-
.../hadoop/ozone/om/OMMultiTenantManager.java | 2 +-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 2 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 143 +
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 186 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 275 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 583 ++++
.../hadoop/ozone/om/OmMetadataReaderMetrics.java | 49 +
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 265 ++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 271 ++
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 163 +
.../hadoop/ozone/om/OpenKeyCleanupService.java | 236 --
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 16 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 121 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 49 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1121 +++---
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 2 -
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 104 -
.../hadoop/ozone/om/S3SecretManagerImpl.java | 93 +-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 89 +-
.../apache/hadoop/ozone/om/SnapshotChainInfo.java | 57 +
.../hadoop/ozone/om/SnapshotChainManager.java | 512 +++
.../hadoop/ozone/om/SstFilteringService.java | 212 ++
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 26 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 109 +-
.../org/apache/hadoop/ozone/om/VolumeManager.java | 2 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 170 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 13 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 14 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 20 +-
.../om/multitenant/OMRangerBGSyncService.java | 870 -----
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 444 ++-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 13 +
.../ozone/om/ratis/OzoneManagerStateMachine.java | 8 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 10 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 2 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 17 +-
.../hadoop/ozone/om/request/OMEchoRPCRequest.java | 41 -
.../om/request/bucket/OMBucketCreateRequest.java | 136 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 56 +
.../request/bucket/OMBucketSetPropertyRequest.java | 50 +-
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 25 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 11 +-
.../request/file/OMFileCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/file/OMFileRequest.java | 41 +-
.../om/request/key/OMAllocateBlockRequest.java | 6 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 3 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 80 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 100 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 71 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 25 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 88 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 37 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../S3MultipartUploadCommitPartRequest.java | 9 +-
.../S3MultipartUploadCompleteRequest.java | 3 +-
.../om/request/s3/security/OMSetSecretRequest.java | 85 +-
.../om/request/s3/security/S3GetSecretRequest.java | 110 +-
.../request/s3/security/S3RevokeSecretRequest.java | 45 +-
.../request/s3/security/S3SecretRequestHelper.java | 2 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 42 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 11 +
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 20 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 190 +
.../request/snapshot/OMSnapshotDeleteRequest.java | 227 ++
.../ozone/om/request/snapshot/package-info.java | 23 +
.../om/request/validation/RequestValidations.java | 25 +-
.../om/request/validation/ValidatorRegistry.java | 7 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 16 +
.../response/file/OMDirectoryCreateResponse.java | 9 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 10 +-
.../response/file/OMFileCreateResponseWithFSO.java | 6 +
.../key/OMDirectoriesPurgeResponseWithFSO.java | 19 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 22 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 14 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 7 +
.../response/key/OMKeyDeleteResponseWithFSO.java | 4 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 45 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 4 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 15 +-
.../response/s3/security/OMSetSecretResponse.java | 12 +-
.../response/s3/security/S3GetSecretResponse.java | 20 +-
.../s3/security/S3RevokeSecretResponse.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdResponse.java | 21 +-
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 13 +-
.../snapshot/OMSnapshotCreateResponse.java | 72 +
.../snapshot/OMSnapshotDeleteResponse.java | 67 +
.../ozone/om/response/snapshot/package-info.java | 23 +
.../ozone/om/service/DirectoryDeletingService.java | 391 +++
.../ozone/om/service/KeyDeletingService.java | 318 ++
.../ozone/om/service/OMRangerBGSyncService.java | 874 +++++
.../ozone/om/service/OpenKeyCleanupService.java | 239 ++
.../hadoop/ozone/om/service/package-info.java | 24 +
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 390 +++
...OzoneManagerProtocolServerSideTranslatorPB.java | 95 +-
.../protocolPB/OzoneManagerRequestHandler.java | 111 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 4 +-
.../hadoop/ozone/security/OMCertificateClient.java | 203 ++
.../OzoneDelegationTokenSecretManager.java | 30 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 22 +-
.../webapps/ozoneManager/om-overview.html | 24 +
.../resources/webapps/ozoneManager/ozoneManager.js | 30 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 2 +-
.../apache/hadoop/ozone/om/TestChunkStreams.java | 150 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 317 --
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 181 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 315 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 225 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 128 +
.../hadoop/ozone/om/TestOpenKeyCleanupService.java | 224 --
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 56 +-
.../org/apache/hadoop/ozone/om/TestScmClient.java | 191 +
.../apache/hadoop/ozone/om/TestSnapshotChain.java | 306 ++
.../apache/hadoop/ozone/om/TestSnapshotInfo.java | 102 +
.../hadoop/ozone/om/TestSstFilteringService.java | 294 ++
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 216 ++
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 101 +
.../ozone/om/request/bucket/TestBucketRequest.java | 5 +
.../request/bucket/TestOMBucketCreateRequest.java | 72 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 73 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 293 +-
.../request/file/TestOMDirectoryCreateRequest.java | 37 +
.../file/TestOMDirectoryCreateRequestWithFSO.java | 39 +
.../om/request/file/TestOMFileCreateRequest.java | 34 +-
.../file/TestOMFileCreateRequestWithFSO.java | 31 +
.../TestOMDirectoriesPurgeRequestAndResponse.java | 297 ++
.../om/request/key/TestOMKeyCommitRequest.java | 130 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 11 +-
.../om/request/key/TestOMKeyCreateRequest.java | 32 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 6 +
.../om/request/key/TestOMKeyRenameRequest.java | 121 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 100 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 9 +-
.../TestS3MultipartUploadCompleteRequest.java | 6 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 6 +
.../s3/security/TestS3GetSecretRequest.java | 219 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 277 ++
.../snapshot/TestOMSnapshotDeleteRequest.java | 331 ++
.../ozone/om/request/snapshot/package-info.java | 23 +
.../volume/TestOMVolumeSetQuotaRequest.java | 23 +
.../file/TestOMDirectoryCreateResponse.java | 19 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 19 +-
.../om/response/key/TestOMKeyCommitResponse.java | 9 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeyRenameResponse.java | 186 +-
.../key/TestOMKeyRenameResponseWithFSO.java | 94 +
.../key/TestOMKeysDeleteResponseWithFSO.java | 7 +-
.../s3/multipart/TestS3MultipartResponse.java | 12 +-
.../TestS3MultipartUploadAbortResponse.java | 3 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 12 +-
.../snapshot/TestOMSnapshotCreateResponse.java | 122 +
.../snapshot/TestOMSnapshotDeleteResponse.java | 125 +
.../ozone/om/response/snapshot/package-info.java | 23 +
.../ozone/om/service/TestKeyDeletingService.java | 448 +++
.../om/service/TestOpenKeyCleanupService.java | 228 ++
.../ozone/om/upgrade/TestOMVersionManager.java | 4 +-
.../hadoop/ozone/security/S3SecretStoreMap.java | 57 +
.../security/TestOmCertificateClientInit.java | 162 +
.../TestOzoneDelegationTokenSecretManager.java | 92 +-
.../security/acl/TestOzoneAdministrators.java | 66 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 3 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 3 +-
hadoop-ozone/ozonefs-common/pom.xml | 13 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 47 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 225 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 236 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 353 +-
.../hadoop/fs/ozone/CapableOzoneFSInputStream.java | 4 +-
.../fs/ozone/CapableOzoneFSOutputStream.java | 61 +
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 57 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 49 +-
.../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 +
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 26 +-
.../hadoop/fs/ozone/OzoneStreamCapabilities.java | 47 -
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 32 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 36 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 20 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 20 +-
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 118 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 10 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 7 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 7 +
hadoop-ozone/ozonefs-shaded/pom.xml | 8 +-
hadoop-ozone/ozonefs/pom.xml | 9 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 7 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 7 +
hadoop-ozone/pom.xml | 23 +-
hadoop-ozone/recon-codegen/pom.xml | 3 +-
hadoop-ozone/recon/pom.xml | 4 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 4 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 137 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 20 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 24 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 23 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 44 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 6 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 52 +
.../ozone/recon/api/filters/ReconAdminFilter.java | 12 +-
.../recon/api/handlers/BucketEntityHandler.java | 36 +-
.../ozone/recon/api/handlers/BucketHandler.java | 60 +-
.../recon/api/handlers/DirectoryEntityHandler.java | 46 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 24 +-
.../ozone/recon/api/handlers/KeyEntityHandler.java | 26 +-
.../recon/api/handlers/LegacyBucketHandler.java | 339 ++
.../recon/api/handlers/RootEntityHandler.java | 56 +-
.../recon/api/handlers/UnknownEntityHandler.java | 8 +-
.../recon/api/handlers/VolumeEntityHandler.java | 32 +-
.../ozone/recon/api/types/BucketObjectDBInfo.java | 168 +
.../recon/api/types/ClusterStateResponse.java | 36 +
.../ozone/recon/api/types/ContainerKeyPrefix.java | 9 +
.../hadoop/ozone/recon/api/types/CountStats.java | 66 +
.../ozone/recon/api/types/KeyObjectDBInfo.java | 154 +
.../ozone/recon/api/types/KeyPrefixContainer.java | 101 +
.../hadoop/ozone/recon/api/types/NSSummary.java | 4 +-
.../recon/api/types/NamespaceSummaryResponse.java | 138 +-
.../hadoop/ozone/recon/api/types/ObjectDBInfo.java | 140 +
.../ozone/recon/api/types/PipelineMetadata.java | 9 +-
.../ozone/recon/api/types/VolumeObjectDBInfo.java | 78 +
.../ozone/recon/fsck/ContainerHealthTask.java | 67 +-
.../ozone/recon/fsck/ReconSafeModeMgrTask.java | 122 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 12 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 33 +-
.../ozone/recon/scm/ReconContainerManager.java | 27 +-
.../recon/scm/ReconDatanodeProtocolServer.java | 2 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 12 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../ozone/recon/scm/ReconSafeModeManager.java | 12 +-
.../ozone/recon/scm/ReconStaleNodeHandler.java | 58 +
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 4 +
.../scm/ReconStorageContainerManagerFacade.java | 250 +-
.../recon/spi/OzoneManagerServiceProvider.java | 6 +
.../recon/spi/ReconContainerMetadataManager.java | 17 +
.../recon/spi/StorageContainerServiceProvider.java | 21 +
.../recon/spi/impl/KeyPrefixContainerCodec.java | 92 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 155 +-
.../impl/ReconContainerMetadataManagerImpl.java | 119 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 12 +-
.../impl/StorageContainerServiceProviderImpl.java | 20 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 21 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 233 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 228 ++
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 71 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 310 ++
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 15 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 17 +
.../resources/webapps/recon/ozone-recon-web/NOTICE | 5 -
.../webapps/recon/ozone-recon-web/api/db.json | 849 ++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 5 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 52 +-
.../src/components/navBar/navBar.tsx | 5 +
.../webapps/recon/ozone-recon-web/src/routes.tsx | 8 +
.../ozone-recon-web/src/utils/autoReloadHelper.tsx | 1 +
.../recon/ozone-recon-web/src/utils/themeIcons.tsx | 6 +-
.../src/views/datanodes/datanodes.tsx | 47 +-
.../src/views/diskUsage/diskUsage.less | 36 +-
.../src/views/diskUsage/diskUsage.tsx | 199 +-
.../src/views/insights/insights.tsx | 10 +-
.../views/missingContainers/missingContainers.tsx | 240 +-
.../src/views/overview/overview.tsx | 72 +-
.../src/views/pipelines/pipelines.less | 9 +-
.../src/views/pipelines/pipelines.tsx | 25 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 28 +-
.../ozone/recon/api/TestContainerEndpoint.java | 68 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 8 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 175 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 1288 +++++++
.../recon/api/TestTotalOpenContainerCount.java | 396 +++
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 143 +
.../ozone/recon/api/filters/TestAdminFilter.java | 25 +
.../hadoop/ozone/recon/common/CommonUtils.java | 225 ++
.../ozone/recon/fsck/TestContainerHealthTask.java | 20 +-
.../TestContainerHealthTaskRecordGenerator.java | 42 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 11 +
.../scm/AbstractReconContainerManagerTest.java | 4 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 70 +-
.../TestReconContainerMetadataManagerImpl.java | 25 +
.../ozone/recon/tasks/TestNSSummaryTask.java | 492 +++
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 21 +-
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 740 ++++
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 12 +-
.../ozone/recon/tasks/TestTableCountTask.java | 2 +-
hadoop-ozone/s3gateway/pom.xml | 50 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 72 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 34 +-
.../ozone/s3/signature/StringToSignProducer.java | 1 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +
.../hadoop/ozone/client/ClientProtocolStub.java | 65 +
.../hadoop/ozone/client/OzoneBucketStub.java | 4 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 106 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 38 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 3 +-
hadoop-ozone/tools/pom.xml | 14 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 7 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 7 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 62 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 9 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 7 +-
.../ozone/admin/nssummary/SummarySubCommand.java | 7 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 2 +-
.../ozone/admin/reconfig/ReconfigureCommands.java | 75 +
.../reconfig/ReconfigurePropertiesSubcommand.java | 53 +
.../admin/reconfig/ReconfigureStartSubcommand.java | 49 +
.../reconfig/ReconfigureStatusSubcommand.java | 89 +
.../admin/reconfig/ReconfigureSubCommandUtil.java | 46 +
.../hadoop/ozone/admin/reconfig/package-info.java | 23 +
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 -
.../hadoop/ozone/debug/DBDefinitionFactory.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 127 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 2 +-
.../ozone/debug/container/ContainerCommands.java | 12 +
.../ozone/debug/container/ExportSubcommand.java | 37 +-
.../ozone/freon/ClosedContainerReplicator.java | 27 +-
.../hadoop/ozone/freon/ContentGenerator.java | 18 +
.../ozone/freon/DatanodeSimulationState.java | 375 ++
.../hadoop/ozone/freon/DatanodeSimulator.java | 587 ++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +
.../ozone/freon/FreonReplicationOptions.java | 2 +-
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 +
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 2 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 29 +-
.../ozone/freon/OzoneClientKeyReadWriteOps.java | 245 ++
.../hadoop/ozone/freon/RangeKeysGenerator.java | 164 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 36 +
.../hadoop/ozone/shell/ReplicationOptions.java | 18 +-
.../java/org/apache/hadoop/ozone/shell/Shell.java | 21 +-
.../ozone/shell/ShellReplicationOptions.java | 3 +-
.../ozone/shell/bucket/ClearQuotaHandler.java | 12 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 2 +-
.../ozone/shell/bucket/ListBucketHandler.java | 17 +-
.../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 39 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 67 +-
.../shell/snapshot/BucketSnapshotHandler.java | 36 +
.../shell/snapshot/CreateSnapshotHandler.java | 62 +
.../shell/snapshot/DeleteSnapshotHandler.java | 61 +
.../ozone/shell/snapshot/ListSnapshotHandler.java | 62 +
.../ozone/shell/snapshot/SnapshotCommands.java | 75 +
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 67 +
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 47 +
.../hadoop/ozone/shell/snapshot/package-info.java | 23 +
.../ozone/shell/volume/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/volume/SetQuotaHandler.java | 11 +-
.../hadoop/ozone/shell/TestOzoneAddress.java | 105 +-
pom.xml | 596 ++--
1305 files changed, 94551 insertions(+), 27605 deletions(-)
create mode 100644 .github/workflows/ci.yml
create mode 100644 .github/workflows/scheduled_ci.yml
delete mode 100644 dev-support/annotations/pom.xml
delete mode 100644 dev-support/annotations/src/main/java/org/apache/ozone/annotations/ReplicateAnnotationProcessor.java
delete mode 100644 dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
create mode 100644 hadoop-hdds/annotations/pom.xml
create mode 100644 hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/ReplicateAnnotationProcessor.java
rename {dev-support => hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java (100%)
rename {dev-support => hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/package-info.java (100%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/AbstractDataStreamOutput.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/PartInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/DuplicatedPipelineIdException.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InvalidPipelineStateException.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CheckedRunnable.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CheckedSupplier.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionSupervisor.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/AbstractContainerScanner.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/AbstractContainerScannerMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScrubberMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScannerMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScrubberMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScannerConfiguration.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScrubberConfiguration.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandScannerMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerUploader.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerCompression.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerResponseStream.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/PushReplicator.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerOutputStream.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java
delete mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestKeyValueStreamDataChannel.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannerConfiguration.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannerMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberConfiguration.java
delete mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberMetrics.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/GrpcOutputStreamTest.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerCompression.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerResponseStream.java
delete mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcOutputStream.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestPushReplicator.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerOutputStream.java
create mode 100644 hadoop-hdds/docs/content/feature/GrafanaOzoneObjectMetrics.png
create mode 100644 hadoop-hdds/docs/content/feature/GrafanaOzoneOverall.png
create mode 100644 hadoop-hdds/docs/content/feature/GrafanaOzoneRPCMetrics.png
create mode 100644 hadoop-hdds/docs/content/feature/Reconfigurability.md
create mode 100644 hadoop-hdds/docs/content/feature/Reconfigurability.zh.md
create mode 100644 hadoop-hdds/docs/content/feature/Streaming-Write-Pipeline.md
create mode 100644 hadoop-hdds/docs/content/tools/LogsInOzone.md
delete mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/package-info.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/ReconfigureProtocol.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolClientSideTranslatorPB.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolPB.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolServerSideTranslatorPB.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/KeyStoresFactory.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509KeyManager.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509TrustManager.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/package-info.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/PKIProfiles/DefaultCAProfile.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/PKIProfiles/DefaultProfile.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/PKIProfiles/PKIProfile.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/PKIProfiles/package-info.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/profile/DefaultCAProfile.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/profile/DefaultProfile.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/profile/PKIProfile.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/profile/package-info.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateSignRequest.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/SelfSignedCertificate.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exception/CertificateException.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exception/package-info.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneAdmins.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IEventInfo.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2Metrics.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/BooleanTriFunction.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DecayRpcSchedulerUtil.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetricsUtil.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/PrometheusMetricsSinkUtil.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TableCacheMetrics.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/UgiMetricsUtil.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheStats.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheStatsRecorder.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestPemFileBasedKeyStoresFactory.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509KeyManager.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509TrustManager.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDnCertificateClientInit.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateSignRequest.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestRootCertificate.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpServer2MetricsTest.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsIntegration.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsSink.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestDecayRpcSchedulerUtil.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestPrometheusMetricsSinkUtil.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestUgiMetricsUtil.java
create mode 100644 hadoop-hdds/interface-client/src/main/proto/ReconfigureProtocol.proto
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/README.md
copy {hadoop-ozone/insight => hadoop-hdds/rocksdb-checkpoint-differ}/dev-support/findbugsExcludeFile.xml (100%)
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/package-info.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/DifferSnapshotInfo.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
copy hadoop-hdds/{container-service/src/main/java/org/apache/hadoop/ozone => rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff}/package-info.java (100%)
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
copy {hadoop-ozone/common => hadoop-hdds/rocksdb-checkpoint-differ}/src/test/resources/log4j.properties (100%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/RatisContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOpsSubscriber.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/DatanodeCommandCountUpdatedHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationQueue.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnhealthyReplicationProcessor.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/AbstractCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/HealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/MismatchedReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/package-info.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/report/ContainerReportValidator.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/report/package-info.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicyFactory.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ContainerReportQueue.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestDatanodeCommandCountUpdatedHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECContainerHealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestECReplicationCheckHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestMismatchedReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/package-info.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/report/TestContainerReportValidator.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/report/package-info.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSCMMetadataStoreImpl.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorTestUtil.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/SpyInputStream.java
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/SpyOutputStream.java
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CleanExpired.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/cert/TestCleanExpired.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ChecksumHelperFactory.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3Batcher.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretCache.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretFunction.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretLockedManager.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretStore.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/package-info.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
create mode 100644 hadoop-ozone/dev-support/intellij/ozone-site-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Datanode1-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Datanode2-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Datanode3-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/OzoneFsShell-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/OzoneManager-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/OzoneManagerInit-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/OzoneShell-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/PrimordialSCM-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/PrimordialSCMInit-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Recon-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Scm2-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Scm2Bootstrap-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Scm3-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/Scm3Bootstrap-ha.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/ScmRoles.xml
copy hadoop-ozone/dist/src/main/compose/{ozone-csi => ozone-legacy-bucket}/.env (100%)
create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md
copy hadoop-ozone/dist/src/main/compose/{ozone => ozone-legacy-bucket}/docker-compose.yaml (100%)
create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config
create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-glyphicons.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-guava.txt
delete mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-net.sf.jopt-simple-jopt-simple.txt
delete mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.html
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/NOTICE-ratis-thirdparty-misc.txt
delete mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/NOTICE-ratis-thirtparty-misc.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/licenses/LICENSE-bootstrap.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/licenses/LICENSE-glyphicons.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/licenses/LICENSE-guava.txt
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/ozonefs.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/recon/recon-nssummary.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-fs.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-setup.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot
delete mode 100644 hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDatanodeProtocolServer.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestScmApplyTransactionFailure.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerReplication.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestDataScanner.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestLDBCli.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmLDBCli.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerBGSyncService.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfigure.java
create mode 100644 hadoop-ozone/integration-test/src/test/resources/testSequenceFile
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBSnapshotInfoCodec.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DirectoryDeletingService.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReaderMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotMetrics.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainInfo.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/OMRangerBGSyncService.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMEchoRPCRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotDeleteResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OMRangerBGSyncService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOpenKeyCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/S3SecretStoreMap.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSOutputStream.java
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java
delete mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java
create mode 100644 hadoop-ozone/ozonefs-hadoop3-client/pom.xml
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyPrefixContainer.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconSafeModeMgrTask.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStaleNodeHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/KeyPrefixContainerCodec.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/NOTICE
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureCommands.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigurePropertiesSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureStartSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureStatusSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureSubCommandUtil.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/package-info.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/BucketSnapshotHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/CreateSnapshotHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/DeleteSnapshotHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotUri.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/package-info.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/04: HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c4a4cc0990d0dc8ea64c2597a4de3449b51a9ab5
Author: Symious <yi...@foxmail.com>
AuthorDate: Wed Sep 7 12:19:57 2022 +0800
HDDS-7155. [DiskBalancer] Create interface between SCM and DN (#3701)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 21 ++--
.../protocol/StorageContainerLocationProtocol.java | 17 ++--
.../scm/storage/DiskBalancerConfiguration.java | 39 +++----
.../common/src/main/resources/ozone-default.xml | 8 ++
.../common/report/DiskBalancerReportPublisher.java | 66 ++++++++++++
.../common/report/ReportPublisherFactory.java | 3 +
.../common/statemachine/DatanodeStateMachine.java | 2 +
.../common/statemachine/StateContext.java | 13 +++
.../commandhandler/DiskBalancerCommandHandler.java | 112 +++++++++++++++++++++
.../states/endpoint/HeartbeatEndpointTask.java | 7 ++
.../ozone/container/ozoneimpl/OzoneContainer.java | 6 ++
.../protocol/commands/DiskBalancerCommand.java | 74 ++++++++++++++
.../common/report/TestReportPublisher.java | 42 ++++++++
.../common/report/TestReportPublisherFactory.java | 12 +++
...inerLocationProtocolClientSideTranslatorPB.java | 81 +++++++++++++--
.../src/main/proto/ScmAdminProtocol.proto | 24 ++++-
.../interface-client/src/main/proto/hdds.proto | 10 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 17 ++++
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 9 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 111 ++++++++++++++------
.../hdds/scm/node/DiskBalancerReportHandler.java | 65 ++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 2 -
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 17 ++++
...inerLocationProtocolServerSideTranslatorPB.java | 52 +++++++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 22 ++--
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 23 +++++
.../hdds/scm/server/StorageContainerManager.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 51 +++++++++-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../hdds/scm/node/TestDiskBalancerManager.java | 34 +++++++
.../testutils/ReplicationNodeManagerMock.java | 6 ++
.../hdds/scm/cli/ContainerOperationClient.java | 29 +++---
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +
35 files changed, 902 insertions(+), 100 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index cb258dfa74..7b3a9c0dd7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -46,6 +46,10 @@ public final class HddsConfigKeys {
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
"60s";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL =
+ "hdds.disk.balancer.report.interval";
+ public static final String HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT =
+ "60s";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 76c07d1c1b..98e6e26e87 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -400,7 +400,8 @@ public interface ScmClient extends Closeable {
throws IOException;
/**
- * Get DiskBalancer status.
+ * Get DiskBalancer report.
+ * REPORT shows the current volume density of datanodes.
* @param count top datanodes that need balancing
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
@@ -410,33 +411,39 @@ public interface ScmClient extends Closeable {
/**
* Get DiskBalancer status.
+ * STATUS shows the running status of DiskBalancer on datanodes.
* @param hosts If hosts is not null, return status of hosts; If hosts is
* null, return status of all datanodes in balancing.
* @return List of DatanodeDiskBalancerInfo.
* @throws IOException
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidth,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 56ed8c7086..afdf36a5f5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -441,26 +441,31 @@ public interface StorageContainerLocationProtocol extends Closeable {
* Get DiskBalancer status.
*/
List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException;
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus,
+ int clientVersion) throws IOException;
/**
* Start DiskBalancer.
*/
- void startDiskBalancer(
+ List<DatanodeAdminError> startDiskBalancer(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
/**
* Stop DiskBalancer.
*/
- void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+ List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException;
/**
* Update DiskBalancer Configuration.
*/
- void updateDiskBalancerConfiguration(
+ List<DatanodeAdminError> updateDiskBalancerConfiguration(
Optional<Double> threshold,
- Optional<Double> bandwidth,
+ Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
index 704b383679..43417f2e81 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
@@ -36,7 +36,7 @@ public final class DiskBalancerConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerConfiguration.class);
- @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ @Config(key = "volume.density.threshold", type = ConfigType.DOUBLE,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "Threshold is a percentage in the range of 0 to 100. A " +
"datanode is considered balanced if for each volume, the " +
@@ -45,12 +45,12 @@ public final class DiskBalancerConfiguration {
" of the entire datanode) no more than the threshold.")
private double threshold = 10d;
- @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.LONG,
defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
description = "The max balance speed.")
- private double diskBandwidth = 10;
+ private long diskBandwidthInMB = 10;
- @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ @Config(key = "parallel.thread", type = ConfigType.INT,
defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
description = "The max parallel balance thread count.")
private int parallelThread = 5;
@@ -86,21 +86,21 @@ public final class DiskBalancerConfiguration {
*
* @return max disk bandwidth per second
*/
- public double getDiskBandwidth() {
- return diskBandwidth;
+ public double getDiskBandwidthInMB() {
+ return diskBandwidthInMB;
}
/**
* Sets the disk bandwidth value for Disk Balancer.
*
- * @param diskBandwidth the bandwidth to control balance speed
+ * @param diskBandwidthInMB the bandwidth to control balance speed
*/
- public void setDiskBandwidth(double diskBandwidth) {
- if (diskBandwidth <= 0d) {
+ public void setDiskBandwidthInMB(long diskBandwidthInMB) {
+ if (diskBandwidthInMB <= 0L) {
throw new IllegalArgumentException(
- "diskBandwidth must be a value larger than 0.");
+ "diskBandwidthInMB must be a value larger than 0.");
}
- this.diskBandwidth = diskBandwidth;
+ this.diskBandwidthInMB = diskBandwidthInMB;
}
/**
@@ -124,6 +124,7 @@ public final class DiskBalancerConfiguration {
}
this.parallelThread = parallelThread;
}
+
@Override
public String toString() {
return String.format("Disk Balancer Configuration values:%n" +
@@ -132,7 +133,7 @@ public final class DiskBalancerConfiguration {
"%-50s %s%n" +
"%-50s %s%n",
"Key", "Value",
- "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidthInMB,
"Parallel Thread", parallelThread);
}
@@ -141,21 +142,21 @@ public final class DiskBalancerConfiguration {
HddsProtos.DiskBalancerConfigurationProto.newBuilder();
builder.setThreshold(threshold)
- .setDiskBandwidth(diskBandwidth)
+ .setDiskBandwidthInMB(diskBandwidthInMB)
.setParallelThread(parallelThread);
return builder;
}
- static DiskBalancerConfiguration fromProtobuf(
+ public static DiskBalancerConfiguration fromProtobuf(
@NotNull HddsProtos.DiskBalancerConfigurationProto proto,
- @NotNull OzoneConfiguration ozoneConfiguration) {
+ @NotNull ConfigurationSource configurationSource) {
DiskBalancerConfiguration config =
- ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ configurationSource.getObject(DiskBalancerConfiguration.class);
if (proto.hasThreshold()) {
config.setThreshold(proto.getThreshold());
}
- if (proto.hasDiskBandwidth()) {
- config.setDiskBandwidth(proto.getDiskBandwidth());
+ if (proto.hasDiskBandwidthInMB()) {
+ config.setDiskBandwidthInMB(proto.getDiskBandwidthInMB());
}
if (proto.hasParallelThread()) {
config.setParallelThread(proto.getParallelThread());
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c6adf0c2db..8e3c78b35b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -365,6 +365,14 @@
datanode periodically send pipeline report to SCM. Unit could be
defined with postfix (ns,ms,s,m,h,d)</description>
</property>
+ <property>
+ <name>hdds.disk.balancer.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, CONTAINER, DISK_BALANCER</tag>
+ <description>Time interval of the datanode to send disk balancer report. Each
+ datanode periodically sends disk balancer report to SCM. Unit could be
+ defined with postfix (ns,ms,s,m,h,d)</description>
+ </property>
<property>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
new file mode 100644
index 0000000000..2bb78c1ee6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/DiskBalancerReportPublisher.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.report;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT;
+
+
+/**
+ * Publishes DiskBalancer report which will be sent to SCM as part of heartbeat.
+ * DiskBalancer Report consist of the following information:
+ * - isBalancerRunning
+ * - balancedBytes
+ * - DiskBalancerConfiguration
+ */
+public class DiskBalancerReportPublisher extends
+ ReportPublisher<DiskBalancerReportProto> {
+
+ private Long diskBalancerReportInterval = null;
+
+ @Override
+ protected long getReportFrequency() {
+ if (diskBalancerReportInterval == null) {
+ diskBalancerReportInterval = getConf().getTimeDuration(
+ HDDS_DISK_BALANCER_REPORT_INTERVAL,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+ getConf());
+
+ Preconditions.checkState(
+ heartbeatFrequency <= diskBalancerReportInterval,
+ HDDS_DISK_BALANCER_REPORT_INTERVAL +
+ " cannot be configured lower than heartbeat frequency " +
+ heartbeatFrequency + ".");
+ }
+ return diskBalancerReportInterval;
+ }
+
+ @Override
+ protected DiskBalancerReportProto getReport() {
+ return getContext().getParent().getContainer().getDiskBalancerReport();
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index 3be1b5e077..19d2806919 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -55,6 +56,8 @@ public class ReportPublisherFactory {
report2publisher.put(PipelineReportsProto.class,
PipelineReportPublisher.class);
report2publisher.put(CRLStatusReport.class, CRLStatusReportPublisher.class);
+ report2publisher.put(DiskBalancerReportProto.class,
+ DiskBalancerReportPublisher.class);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 01d114245a..86d92122cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.Comm
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CreatePipelineCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DiskBalancerCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.FinalizeNewLayoutVersionCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ReconstructECContainersCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.RefreshVolumeUsageCommandHandler;
@@ -228,6 +229,7 @@ public class DatanodeStateMachine implements Closeable {
.addHandler(new SetNodeOperationalStateCommandHandler(conf))
.addHandler(new FinalizeNewLayoutVersionCommandHandler())
.addHandler(new RefreshVolumeUsageCommandHandler())
+ .addHandler(new DiskBalancerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index c4d3428a80..c75c7a455d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
@@ -98,6 +99,9 @@ public class StateContext {
@VisibleForTesting
static final String CRL_STATUS_REPORT_PROTO_NAME =
CRLStatusReport.getDescriptor().getFullName();
+ @VisibleForTesting
+ static final String DISK_BALANCER_REPORT_PROTO_NAME =
+ DiskBalancerReportProto.getDescriptor().getFullName();
static final Logger LOG =
LoggerFactory.getLogger(StateContext.class);
@@ -113,6 +117,7 @@ public class StateContext {
private final AtomicReference<Message> nodeReport;
private final AtomicReference<Message> pipelineReports;
private final AtomicReference<Message> crlStatusReport;
+ private final AtomicReference<Message> diskBalancerReport;
// Incremental reports are queued in the map below
private final Map<InetSocketAddress, List<Message>>
incrementalReportsQueue;
@@ -172,6 +177,7 @@ public class StateContext {
nodeReport = new AtomicReference<>();
pipelineReports = new AtomicReference<>();
crlStatusReport = new AtomicReference<>(); // Certificate Revocation List
+ diskBalancerReport = new AtomicReference<>();
endpoints = new HashSet<>();
containerActions = new HashMap<>();
pipelineActions = new HashMap<>();
@@ -197,6 +203,8 @@ public class StateContext {
type2Reports.put(PIPELINE_REPORTS_PROTO_NAME, pipelineReports);
fullReportTypeList.add(CRL_STATUS_REPORT_PROTO_NAME);
type2Reports.put(CRL_STATUS_REPORT_PROTO_NAME, crlStatusReport);
+ fullReportTypeList.add(DISK_BALANCER_REPORT_PROTO_NAME);
+ type2Reports.put(DISK_BALANCER_REPORT_PROTO_NAME, diskBalancerReport);
}
/**
@@ -929,6 +937,11 @@ public class StateContext {
return crlStatusReport.get();
}
+ @VisibleForTesting
+ public Message getDiskBalancerReport() {
+ return diskBalancerReport.get();
+ }
+
public void configureReconHeartbeatFrequency() {
reconHeartbeatFrequency.set(getReconHeartbeatInterval(conf));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
new file mode 100644
index 0000000000..705c0af0e1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DiskBalancerCommandHandler.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Handler for DiskBalancer command received from SCM.
+ */
+public class DiskBalancerCommandHandler implements CommandHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerCommandHandler.class);
+
+ private AtomicLong invocationCount = new AtomicLong(0);
+ private long totalTime;
+
+ /**
+ * Constructs a diskBalancerCommand handler.
+ */
+ public DiskBalancerCommandHandler() {
+ }
+
+ /**
+ * Handles a given SCM command.
+ *
+ * @param command - SCM Command
+ * @param ozoneContainer - Ozone Container.
+ * @param context - Current Context.
+ * @param connectionManager - The SCMs that we are talking to.
+ */
+ @Override
+ public void handle(SCMCommand command, OzoneContainer ozoneContainer,
+ StateContext context, SCMConnectionManager connectionManager) {
+ invocationCount.incrementAndGet();
+ // TODO: Do start/stop/update operation
+ }
+
+ /**
+ * Returns the command type that this command handler handles.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getCommandType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ /**
+ * Returns number of times this handler has been invoked.
+ *
+ * @return int
+ */
+ @Override
+ public int getInvocationCount() {
+ return (int)invocationCount.get();
+ }
+
+ /**
+ * Returns the average time this function takes to run.
+ *
+ * @return long
+ */
+ @Override
+ public long getAverageRunTime() {
+ if (invocationCount.get() > 0) {
+ return totalTime / invocationCount.get();
+ }
+ return 0;
+ }
+
+ @Override
+ public int getQueuedCount() {
+ return 0;
+ }
+
+ private void startDiskBalancer(DiskBalancerConfiguration configuration) {
+ // Todo: add implementation to start DiskBalancer
+ }
+
+ private void stopDiskBalancer() {
+ // Todo: add implementation to stop DiskBalancer
+ }
+
+ private void updateDiskBalancer(DiskBalancerConfiguration
+ configuration) {
+ // Todo: add implementation to update diskBalancer configuration
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 3a1bd8ffb3..ac5a14b6aa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.DiskBalancerCommand;
import org.apache.hadoop.ozone.protocol.commands.FinalizeNewLayoutVersionCommand;
import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
import org.apache.hadoop.ozone.protocol.commands.RefreshVolumeUsageCommand;
@@ -418,6 +419,12 @@ public class HeartbeatEndpointTask
commandResponseProto.getRefreshVolumeUsageCommandProto());
processCommonCommand(commandResponseProto, refreshVolumeUsageCommand);
break;
+ case diskBalancerCommand:
+ DiskBalancerCommand diskBalancerCommand =
+ DiskBalancerCommand.getFromProtobuf(
+ commandResponseProto.getDiskBalancerCommandProto(), conf);
+ processCommonCommand(commandResponseProto, diskBalancerCommand);
+ break;
default:
throw new IllegalArgumentException("Unknown response : "
+ commandResponseProto.getCommandType().name());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index b3db557b6f..14a89bb867 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
@@ -521,4 +522,9 @@ public class OzoneContainer {
StorageVolumeChecker getVolumeChecker(ConfigurationSource conf) {
return new StorageVolumeChecker(conf, new Timer());
}
+
+ public DiskBalancerReportProto getDiskBalancerReport() {
+ // TODO: Return real disk balancer report
+ return null;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
new file mode 100644
index 0000000000..ea97f34ca0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DiskBalancerCommand.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerCommandProto;
+
+/**
+ * Informs a datanode to update DiskBalancer status.
+ */
+public class DiskBalancerCommand extends SCMCommand<DiskBalancerCommandProto> {
+
+ private final boolean shouldRun;
+ private final DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public DiskBalancerCommand(final boolean shouldRun,
+ final DiskBalancerConfiguration diskBalancerConfiguration) {
+ this.shouldRun = shouldRun;
+ this.diskBalancerConfiguration = diskBalancerConfiguration;
+ }
+
+ /**
+ * Returns the type of this command.
+ *
+ * @return Type
+ */
+ @Override
+ public SCMCommandProto.Type getType() {
+ return SCMCommandProto.Type.diskBalancerCommand;
+ }
+
+ @Override
+ public DiskBalancerCommandProto getProto() {
+ return DiskBalancerCommandProto.newBuilder()
+ .setShouldRun(shouldRun)
+ .setDiskBalancerConf(diskBalancerConfiguration.toProtobufBuilder())
+ .build();
+ }
+
+ public static DiskBalancerCommand getFromProtobuf(DiskBalancerCommandProto
+ diskbalancerCommandProto, ConfigurationSource configuration) {
+ Preconditions.checkNotNull(diskbalancerCommandProto);
+ return new DiskBalancerCommand(diskbalancerCommandProto.getShouldRun(),
+ DiskBalancerConfiguration.fromProtobuf(
+ diskbalancerCommandProto.getDiskBalancerConf(), configuration));
+ }
+
+ public boolean isShouldRun() {
+ return shouldRun;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index d611caf126..c09261d4ab 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -31,7 +31,9 @@ import org.apache.hadoop.hdds.HddsIdFactory;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
import org.apache.hadoop.hdds.protocol.proto.
@@ -39,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto.
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.junit.jupiter.api.Assertions;
@@ -215,6 +218,45 @@ public class TestReportPublisher {
executorService.shutdown();
}
+ @Test
+ public void testDiskBalancerReportPublisher() throws IOException {
+ StateContext dummyContext = Mockito.mock(StateContext.class);
+ DatanodeStateMachine dummyStateMachine =
+ Mockito.mock(DatanodeStateMachine.class);
+ OzoneContainer dummyContainer = Mockito.mock(OzoneContainer.class);
+
+ DiskBalancerReportProto.Builder builder =
+ DiskBalancerReportProto.newBuilder();
+ builder.setIsRunning(true);
+ builder.setBalancedBytes(1L);
+ builder.setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build());
+ DiskBalancerReportProto dummyReport = builder.build();
+
+ ReportPublisher publisher = new DiskBalancerReportPublisher();
+ when(dummyContext.getParent()).thenReturn(dummyStateMachine);
+ when(dummyStateMachine.getContainer()).thenReturn(dummyContainer);
+ when(dummyContainer.getDiskBalancerReport()).thenReturn(dummyReport);
+ publisher.setConf(config);
+
+ ScheduledExecutorService executorService = HadoopExecutors
+ .newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("Unit test ReportManager Thread - %d").build());
+ publisher.init(dummyContext, executorService);
+ Message report =
+ ((DiskBalancerReportPublisher) publisher).getReport();
+ Assert.assertNotNull(report);
+ for (Descriptors.FieldDescriptor descriptor :
+ report.getDescriptorForType().getFields()) {
+ if (descriptor.getNumber() ==
+ DiskBalancerReportProto.ISRUNNING_FIELD_NUMBER) {
+ Assert.assertEquals(true, report.getField(descriptor));
+ }
+ }
+ executorService.shutdown();
+ }
+
/**
* Get a datanode details.
*
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
index ea915f98a2..0b43bb93f9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.junit.Assert;
import org.junit.Rule;
@@ -66,6 +67,17 @@ public class TestReportPublisherFactory {
Assert.assertEquals(conf, publisher.getConf());
}
+ @Test
+ public void testGetDiskBalancerReportPublisher() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ ReportPublisherFactory factory = new ReportPublisherFactory(conf);
+ ReportPublisher publisher = factory
+ .getPublisherFor(DiskBalancerReportProto.class);
+ Assert.assertEquals(DiskBalancerReportPublisher.class,
+ publisher.getClass());
+ Assert.assertEquals(conf, publisher.getConf());
+ }
+
@Test
public void testInvalidReportPublisher() {
OzoneConfiguration conf = new OzoneConfiguration();
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 0d35798352..936e31d760 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -45,6 +45,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1052,11 +1055,14 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
DatanodeDiskBalancerInfoRequestProto.newBuilder()
.setInfoType(DatanodeDiskBalancerInfoType.status);
hosts.ifPresent(requestBuilder::addAllHosts);
+ status.ifPresent(requestBuilder::setStatus);
DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
DatanodeDiskBalancerInfoResponseProto response =
@@ -1068,22 +1074,83 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.start)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.stop);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
+ HddsProtos.DiskBalancerConfigurationProto.Builder confBuilder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+ threshold.ifPresent(confBuilder::setThreshold);
+ bandwidthInMB.ifPresent(confBuilder::setDiskBandwidthInMB);
+ parallelThread.ifPresent(confBuilder::setParallelThread);
+
+ DatanodeDiskBalancerOpRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerOpRequestProto.newBuilder()
+ .setOpType(DatanodeDiskBalancerOpType.update)
+ .setConf(confBuilder);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+
+ DatanodeDiskBalancerOpResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerOp,
+ builder -> builder.setDatanodeDiskBalancerOpRequest(
+ requestBuilder.build()))
+ .getDatanodeDiskBalancerOpResponse();
+
+ List<DatanodeAdminError> errors = new ArrayList<>();
+ for (DatanodeAdminErrorResponseProto e : response.getFailedHostsList()) {
+ errors.add(new DatanodeAdminError(e.getHost(), e.getError()));
+ }
+ return errors;
}
@Override
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 766db3429c..d6c552c4b4 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -78,7 +78,8 @@ message ScmContainerLocationRequest {
optional GetContainerReplicasRequestProto getContainerReplicasRequest = 39;
optional ReplicationManagerReportRequestProto replicationManagerReportRequest = 40;
optional ResetDeletedBlockRetryCountRequestProto resetDeletedBlockRetryCountRequest = 41;
- optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 42;
+ optional DatanodeDiskBalancerInfoRequestProto datanodeDiskBalancerInfoRequest = 42;
+ optional DatanodeDiskBalancerOpRequestProto datanodeDiskBalancerOpRequest = 43;
}
message ScmContainerLocationResponse {
@@ -128,7 +129,8 @@ message ScmContainerLocationResponse {
optional GetContainerReplicasResponseProto getContainerReplicasResponse = 39;
optional ReplicationManagerReportResponseProto getReplicationManagerReportResponse = 40;
optional ResetDeletedBlockRetryCountResponseProto resetDeletedBlockRetryCountResponse = 41;
- optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 42;
+ optional DatanodeDiskBalancerInfoResponseProto datanodeDiskBalancerInfoResponse = 42;
+ optional DatanodeDiskBalancerOpResponseProto datanodeDiskBalancerOpResponse = 43;
enum Status {
OK = 1;
@@ -178,6 +180,7 @@ enum Type {
ResetDeletedBlockRetryCount = 36;
GetClosedContainerCount = 37;
DatanodeDiskBalancerInfo= 38;
+ DatanodeDiskBalancerOp = 39;
}
/**
@@ -341,12 +344,29 @@ message DatanodeDiskBalancerInfoRequestProto {
required DatanodeDiskBalancerInfoType infoType = 1;
optional uint32 count = 2;
repeated string hosts = 3;
+ optional DiskBalancerRunningStatus status = 4;
}
message DatanodeDiskBalancerInfoResponseProto {
repeated DatanodeDiskBalancerInfoProto info = 1;
}
+enum DatanodeDiskBalancerOpType{
+ start = 1;
+ stop = 2;
+ update = 3;
+}
+
+message DatanodeDiskBalancerOpRequestProto {
+ required DatanodeDiskBalancerOpType opType = 1;
+ repeated string hosts = 2;
+ optional DiskBalancerConfigurationProto conf = 3;
+}
+
+message DatanodeDiskBalancerOpResponseProto {
+ repeated DatanodeAdminErrorResponseProto failedHosts = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 7cee2f6160..231e105aea 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -455,13 +455,19 @@ message ContainerBalancerConfigurationProto {
message DiskBalancerConfigurationProto {
optional double threshold = 1;
- optional double diskBandwidth = 2;
+ optional uint64 diskBandwidthInMB = 2;
optional int32 parallelThread = 3;
}
+enum DiskBalancerRunningStatus {
+ RUNNING = 1;
+ STOPPED = 2;
+ UNKNOWN = 3;
+}
+
message DatanodeDiskBalancerInfoProto {
required DatanodeDetailsProto node = 1;
required double currentVolumeDensitySum = 2;
- optional bool diskBalancerRunning = 3;
+ optional DiskBalancerRunningStatus runningStatus = 3;
optional DiskBalancerConfigurationProto diskBalancerConf = 4;
}
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
index 665665ee38..81e9ee54d3 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
@@ -139,6 +139,7 @@ message SCMHeartbeatRequestProto {
optional PipelineReportsProto pipelineReports = 8;
optional LayoutVersionProto dataNodeLayoutVersion = 9;
optional CommandQueueReportProto commandQueueReport = 10;
+ optional DiskBalancerReportProto diskBalancerReport = 11;
}
message CommandQueueReportProto {
@@ -325,6 +326,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommand = 9;
refreshVolumeUsageInfo = 10;
reconstructECContainersCommand = 11;
+ diskBalancerCommand = 12;
}
// TODO: once we start using protoc 3.x, refactor this message using "oneof"
required Type commandType = 1;
@@ -340,6 +342,7 @@ message SCMCommandProto {
finalizeNewLayoutVersionCommandProto = 10;
optional RefreshVolumeUsageCommandProto refreshVolumeUsageCommandProto = 11;
optional ReconstructECContainersCommandProto reconstructECContainersCommandProto = 12;
+ optional DiskBalancerCommandProto diskBalancerCommandProto = 13;
// If running upon Ratis, holds term of underlying RaftServer iff current
@@ -442,6 +445,14 @@ message DatanodeDetailsAndReplicaIndexProto {
required int32 replicaIndex = 2;
}
+/**
+This command asks the datanode to update diskBalancer status
+ */
+message DiskBalancerCommandProto {
+ required bool shouldRun = 1;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 2;
+}
+
/**
This command asks the datanode to create a pipeline.
*/
@@ -480,6 +491,12 @@ message CRLStatusReport {
repeated int64 pendingCrlIds=2;
}
+message DiskBalancerReportProto {
+ required bool isRunning = 1;
+ optional uint64 balancedBytes = 2;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 3;
+}
+
/**
* This command asks the datanode to process a new CRL.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 639c884795..16c995e1eb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatu
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
@@ -227,6 +228,14 @@ public final class SCMEvents {
new TypedEvent<>(CRLStatusReportFromDatanode.class,
"Crl_Status_Report");
+ /**
+ * DiskBalancer reports are send out by Datanodes. This report is received by
+ * SCMDatanodeHeartbeatDispatcher and DiskBalancer_Report Event is generated.
+ */
+ public static final TypedEvent<DiskBalancerReportFromDatanode>
+ DISK_BALANCER_REPORT = new TypedEvent<>(
+ DiskBalancerReportFromDatanode.class, "DiskBalancer_Report");
+
/**
* Private Ctor. Never Constructed.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
index 5ca66e9eef..c6114c52f5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -22,7 +22,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -47,10 +48,13 @@ public class DiskBalancerManager {
public static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private OzoneConfiguration conf;
private final EventPublisher scmNodeEventPublisher;
private final SCMContext scmContext;
private final NodeManager nodeManager;
private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private Map<DatanodeDetails, Long> balancedBytesMap;
private boolean useHostnames;
/**
@@ -60,6 +64,7 @@ public class DiskBalancerManager {
EventPublisher eventPublisher,
SCMContext scmContext,
NodeManager nodeManager) {
+ this.conf = conf;
this.scmNodeEventPublisher = eventPublisher;
this.scmContext = scmContext;
this.nodeManager = nodeManager;
@@ -95,42 +100,60 @@ public class DiskBalancerManager {
* If hosts is null, return status of all datanodes in balancing.
*/
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
- List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
- new ArrayList<>();
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
List<DatanodeDetails> filterDns = null;
if (hosts.isPresent() && !hosts.get().isEmpty()) {
filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
useHostnames);
}
- for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
- HddsProtos.NodeState.HEALTHY)) {
- if (shouldReturnDatanode(filterDns, datanodeDetails)) {
- double volumeDensitySum =
- getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
- statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
- .setCurrentVolumeDensitySum(volumeDensitySum)
- .setDiskBalancerRunning(isRunning(datanodeDetails))
- .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
- DiskBalancerStatus.DUMMY_STATUS)
- .getDiskBalancerConfiguration().toProtobufBuilder())
- .setNode(datanodeDetails.toProto(clientVersion))
- .build());
- }
+ // Filter Running Status by default
+ HddsProtos.DiskBalancerRunningStatus filterStatus = status.orElse(null);
+
+ if (filterDns != null) {
+ return filterDns.stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(nodeManager::getDatanodeInfo)
+ .map(dn -> getInfoProto(dn, clientVersion))
+ .collect(Collectors.toList());
+ } else {
+ return nodeManager.getAllNodes().stream()
+ .filter(dn -> shouldReturnDatanode(filterStatus, dn))
+ .map(dn -> getInfoProto((DatanodeInfo)dn, clientVersion))
+ .collect(Collectors.toList());
}
- return statusList;
}
- private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ private boolean shouldReturnDatanode(
+ HddsProtos.DiskBalancerRunningStatus status,
DatanodeDetails datanodeDetails) {
- if (hosts == null || hosts.isEmpty()) {
- return isRunning(datanodeDetails);
- } else {
- return hosts.contains(datanodeDetails);
+ boolean shouldReturn = true;
+ // If status specified, do not return if status not match.
+ if (status != null && getRunningStatus(datanodeDetails) != status) {
+ shouldReturn = false;
}
+ return shouldReturn;
}
+ private HddsProtos.DatanodeDiskBalancerInfoProto getInfoProto(
+ DatanodeInfo dn, int clientVersion) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(dn);
+ HddsProtos.DiskBalancerRunningStatus runningStatus =
+ getRunningStatus(dn);
+ HddsProtos.DatanodeDiskBalancerInfoProto.Builder builder =
+ HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(dn.toProto(clientVersion))
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setRunningStatus(getRunningStatus(dn));
+ if (runningStatus != HddsProtos.DiskBalancerRunningStatus.UNKNOWN) {
+ builder.setDiskBalancerConf(statusMap.get(dn)
+ .getDiskBalancerConfiguration().toProtobufBuilder());
+ }
+ return builder.build();
+ }
/**
* Get volume density for a specific DatanodeDetails node.
*
@@ -144,8 +167,7 @@ public class DiskBalancerManager {
DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
double totalCapacity = 0d, totalUsed = 0d;
- for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
- datanodeInfo.getStorageReports()) {
+ for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
totalCapacity += reportProto.getCapacity();
totalUsed += reportProto.getScmUsed();
}
@@ -162,10 +184,17 @@ public class DiskBalancerManager {
return volumeDensitySum;
}
- private boolean isRunning(DatanodeDetails datanodeDetails) {
- return statusMap
- .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
- .isRunning();
+ private HddsProtos.DiskBalancerRunningStatus getRunningStatus(
+ DatanodeDetails datanodeDetails) {
+ if (!statusMap.containsKey(datanodeDetails)) {
+ return HddsProtos.DiskBalancerRunningStatus.UNKNOWN;
+ } else {
+ if (statusMap.get(datanodeDetails).isRunning()) {
+ return HddsProtos.DiskBalancerRunningStatus.RUNNING;
+ } else {
+ return HddsProtos.DiskBalancerRunningStatus.STOPPED;
+ }
+ }
}
@VisibleForTesting
@@ -173,4 +202,24 @@ public class DiskBalancerManager {
statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
new DiskBalancerConfiguration()));
}
-}
\ No newline at end of file
+
+ public void processDiskBalancerReport(DiskBalancerReportProto reportProto,
+ DatanodeDetails dn) {
+ boolean isRunning = reportProto.getIsRunning();
+ DiskBalancerConfiguration diskBalancerConfiguration =
+ reportProto.hasDiskBalancerConf() ?
+ DiskBalancerConfiguration.fromProtobuf(
+ reportProto.getDiskBalancerConf(), conf) :
+ new DiskBalancerConfiguration();
+ statusMap.put(dn, new DiskBalancerStatus(isRunning,
+ diskBalancerConfiguration));
+ if (reportProto.hasBalancedBytes()) {
+ balancedBytesMap.put(dn, reportProto.getBalancedBytes());
+ }
+ }
+
+ @VisibleForTesting
+ public Map<DatanodeDetails, DiskBalancerStatus> getStatusMap() {
+ return statusMap;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
new file mode 100644
index 0000000000..47158972ec
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerReportHandler.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles DiskBalancer Reports from datanode.
+ */
+public class DiskBalancerReportHandler implements
+ EventHandler<DiskBalancerReportFromDatanode> {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(DiskBalancerReportHandler.class);
+
+ private DiskBalancerManager diskBalancerManager;
+
+ public DiskBalancerReportHandler(DiskBalancerManager diskBalancerManager) {
+ this.diskBalancerManager = diskBalancerManager;
+ }
+
+ @Override
+ public void onMessage(DiskBalancerReportFromDatanode reportFromDatanode,
+ EventPublisher publisher) {
+ Preconditions.checkNotNull(reportFromDatanode);
+ DatanodeDetails dn = reportFromDatanode.getDatanodeDetails();
+ DiskBalancerReportProto diskBalancerReportProto =
+ reportFromDatanode.getReport();
+ Preconditions.checkNotNull(dn,
+ "DiskBalancer Report is missing DatanodeDetails.");
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("Processing diskBalancer report for dn: {}", dn);
+ }
+ try {
+ diskBalancerManager.processDiskBalancerReport(
+ diskBalancerReportProto, dn);
+ } catch (Exception e) {
+ LOGGER.error("Failed to process diskBalancer report={} from dn={}.",
+ diskBalancerReportProto, dn, e);
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
index ed22e80e3c..65f577bcee 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -32,8 +32,6 @@ public class DiskBalancerStatus {
private boolean isRunning;
private DiskBalancerConfiguration diskBalancerConfiguration;
- public static final DiskBalancerStatus DUMMY_STATUS =
- new DiskBalancerStatus(false, new DiskBalancerConfiguration());
public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
this.isRunning = isRunning;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 8f72375bcd..c1dffe7f4b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -162,6 +162,14 @@ public interface NodeManager extends StorageContainerNodeProtocol,
*/
DatanodeUsageInfo getUsageInfo(DatanodeDetails dn);
+ /**
+ * Get the datanode info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeInfo of the specified datanode
+ */
+ DatanodeInfo getDatanodeInfo(DatanodeDetails dn);
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails DatanodeDetails.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 7a62088ba7..935bfbc6a8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -860,6 +860,23 @@ public class SCMNodeManager implements NodeManager {
return usageInfo;
}
+ /**
+ * Get the usage info of a specified datanode.
+ *
+ * @param dn the usage of which we want to get
+ * @return DatanodeUsageInfo of the specified datanode
+ */
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ try {
+ return nodeStateManager.getNode(dn);
+ } catch (NodeNotFoundException e) {
+ LOG.warn("Cannot retrieve DatanodeInfo, datanode {} not found.",
+ dn.getUuid());
+ return null;
+ }
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 2f50456211..38a2503b55 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerOpResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -672,6 +673,13 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
request.getDatanodeDiskBalancerInfoRequest(),
request.getVersion()))
.build();
+ case DatanodeDiskBalancerOp:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerOpResponse(getDatanodeDiskBalancerOp(
+ request.getDatanodeDiskBalancerOpRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1183,6 +1191,7 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
request.getTransactionIdList()))
.build();
}
+
public DatanodeDiskBalancerInfoResponseProto getDatanodeDiskBalancerInfo(
StorageContainerLocationProtocolProtos.
DatanodeDiskBalancerInfoRequestProto request, int clientVersion)
@@ -1195,7 +1204,9 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
break;
case status:
infoProtoList = impl.getDiskBalancerStatus(
- Optional.of(request.getHostsList()), clientVersion);
+ Optional.of(request.getHostsList()),
+ Optional.of(request.getStatus()),
+ clientVersion);
break;
default:
infoProtoList = null;
@@ -1204,4 +1215,43 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
.addAllInfo(infoProtoList)
.build();
}
+
+ public DatanodeDiskBalancerOpResponseProto getDatanodeDiskBalancerOp(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerOpRequestProto request)
+ throws IOException {
+ List<DatanodeAdminError> errors;
+ switch (request.getOpType()) {
+ case start:
+ errors = impl.startDiskBalancer(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case update:
+ errors = impl.updateDiskBalancerConfiguration(
+ Optional.of(request.getConf().getThreshold()),
+ Optional.of(request.getConf().getDiskBandwidthInMB()),
+ Optional.of(request.getConf().getParallelThread()),
+ Optional.of(request.getHostsList()));
+ break;
+ case stop:
+ errors = impl.stopDiskBalancer(Optional.of(request.getHostsList()));
+ break;
+ default:
+ errors = new ArrayList<>();
+ }
+
+ DatanodeDiskBalancerOpResponseProto.Builder response =
+ DatanodeDiskBalancerOpResponseProto.newBuilder();
+ for (DatanodeAdminError e : errors) {
+ DatanodeAdminErrorResponseProto.Builder error =
+ DatanodeAdminErrorResponseProto.newBuilder();
+ error.setHost(e.getHostname());
+ error.setError(e.getError());
+ response.addFailedHosts(error);
+ }
+ return response.build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index e9759da960..d3f23089be 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1142,7 +1142,9 @@ public class SCMClientProtocolServer implements
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts, int clientVersion) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> status,
+ int clientVersion) throws IOException {
// check admin authorisation
try {
getScm().checkAdminAccess(getRemoteUser());
@@ -1151,29 +1153,33 @@ public class SCMClientProtocolServer implements
throw e;
}
- return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts, status,
clientVersion);
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidthInMB,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
+ return null;
}
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index 02e1b3fc61..35e15e4efd 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
@@ -58,6 +59,7 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DISK_BALANCER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents
.INCREMENTAL_CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NODE_REPORT;
@@ -212,6 +214,15 @@ public final class SCMDatanodeHeartbeatDispatcher {
commandStatusReport));
}
}
+
+ if (heartbeat.hasDiskBalancerReport()) {
+ LOG.debug("Dispatching DiskBalancer Report.");
+ eventPublisher.fireEvent(
+ DISK_BALANCER_REPORT,
+ new DiskBalancerReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getDiskBalancerReport()));
+ }
}
return commands;
@@ -445,4 +456,16 @@ public final class SCMDatanodeHeartbeatDispatcher {
super(datanodeDetails, report);
}
}
+
+ /**
+ * DiskBalancer report event payload with origin.
+ */
+ public static class DiskBalancerReportFromDatanode
+ extends ReportFromDatanode<DiskBalancerReportProto> {
+
+ public DiskBalancerReportFromDatanode(DatanodeDetails datanodeDetails,
+ DiskBalancerReportProto report) {
+ super(datanodeDetails, report);
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index b8727173f1..f4317ec9b2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerReportHandler;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
@@ -454,6 +455,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PipelineActionHandler(pipelineManager, scmContext, configuration);
CRLStatusReportHandler crlStatusReportHandler =
new CRLStatusReportHandler(certificateStore, configuration);
+ DiskBalancerReportHandler diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
@@ -528,7 +531,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
eventQueue.addHandler(SCMEvents.CRL_STATUS_REPORT, crlStatusReportHandler);
-
+ eventQueue.addHandler(SCMEvents.DISK_BALANCER_REPORT,
+ diskBalancerReportHandler);
}
private void initializeCertificateClient() {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index e1eaf251f5..9cb923c078 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -346,7 +346,36 @@ public class MockNodeManager implements NodeManager {
*/
@Override
public List<DatanodeDetails> getAllNodes() {
- return new ArrayList<>(nodeMetricMap.keySet());
+ // mock storage reports for TestDiskBalancer
+ List<DatanodeDetails> healthyNodesWithInfo = new ArrayList<>();
+ for (Map.Entry<DatanodeDetails, SCMNodeStat> entry:
+ nodeMetricMap.entrySet()) {
+ NodeStatus nodeStatus = NodeStatus.inServiceHealthy();
+ if (staleNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceStale();
+ } else if (deadNodes.contains(entry.getKey())) {
+ nodeStatus = NodeStatus.inServiceDead();
+ }
+ DatanodeInfo di = new DatanodeInfo(entry.getKey(), nodeStatus,
+ UpgradeUtils.defaultLayoutVersionProto());
+
+ long capacity = entry.getValue().getCapacity().get();
+ long used = entry.getValue().getScmUsed().get();
+ long remaining = entry.getValue().getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+
+ healthyNodesWithInfo.add(di);
+ }
+ return healthyNodesWithInfo;
}
/**
@@ -409,6 +438,26 @@ public class MockNodeManager implements NodeManager {
return new DatanodeUsageInfo(datanodeDetails, stat);
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dd) {
+ DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(),
+ UpgradeUtils.defaultLayoutVersionProto());
+ long capacity = nodeMetricMap.get(dd).getCapacity().get();
+ long used = nodeMetricMap.get(dd).getScmUsed().get();
+ long remaining = nodeMetricMap.get(dd).getRemaining().get();
+ StorageReportProto storage1 = HddsTestUtils.createStorageReport(
+ di.getUuid(), "/data1-" + di.getUuidString(),
+ capacity, used, remaining, null);
+ MetadataStorageReportProto metaStorage1 =
+ HddsTestUtils.createMetadataStorageReport(
+ "/metadata1-" + di.getUuidString(), capacity, used,
+ remaining, null);
+ di.updateStorageReports(new ArrayList<>(Arrays.asList(storage1)));
+ di.updateMetaDataStorageReports(
+ new ArrayList<>(Arrays.asList(metaStorage1)));
+ return di;
+ }
+
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails - datanode details.
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
index 22e01b9770..16f013b63f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java
@@ -244,6 +244,11 @@ public class SimpleMockNodeManager implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
return null;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
index 541c9764b7..7005dea292 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -21,8 +21,10 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DiskBalancerReportProto;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.DiskBalancerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.ozone.test.GenericTestUtils;
@@ -33,6 +35,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
+import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class TestDiskBalancerManager {
private NodeManager nodeManager;
private OzoneConfiguration conf;
private String storageDir;
+ private DiskBalancerReportHandler diskBalancerReportHandler;
+ private Random random;
@BeforeEach
public void setup() throws Exception {
@@ -56,6 +61,9 @@ public class TestDiskBalancerManager {
nodeManager = new MockNodeManager(true, 3);
diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
SCMContext.emptyContext(), nodeManager);
+ diskBalancerReportHandler =
+ new DiskBalancerReportHandler(diskBalancerManager);
+ random = new Random();
}
@Test
@@ -73,6 +81,7 @@ public class TestDiskBalancerManager {
@Test
public void testDatanodeDiskBalancerStatus() throws IOException {
diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(1));
// Simulate users asking all status of 3 datanodes
List<String> dns = nodeManager.getAllNodes().stream().map(
@@ -81,6 +90,7 @@ public class TestDiskBalancerManager {
List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(3, statusProtoList.size());
@@ -92,8 +102,32 @@ public class TestDiskBalancerManager {
statusProtoList =
diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ Optional.empty(),
ClientVersion.CURRENT_VERSION);
Assertions.assertEquals(1, statusProtoList.size());
}
+
+ @Test
+ public void testHandleDiskBalancerReportFromDatanode() {
+ for (DatanodeDetails dn: nodeManager.getAllNodes()) {
+ diskBalancerReportHandler.onMessage(
+ new DiskBalancerReportFromDatanode(dn, generateRandomReport()), null);
+ }
+
+ Assertions.assertEquals(3, diskBalancerManager.getStatusMap().size());
+ }
+
+ private DiskBalancerReportProto generateRandomReport() {
+ return DiskBalancerReportProto.newBuilder()
+ .setIsRunning(random.nextBoolean())
+ .setBalancedBytes(random.nextInt(10000))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder()
+ .setThreshold(random.nextInt(99))
+ .setParallelThread(random.nextInt(4) + 1)
+ .setDiskBandwidthInMB(random.nextInt(99) + 1)
+ .build())
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 436bccb09d..2be5da3655 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -189,6 +190,11 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
+ return null;
+ }
+
/**
* Return the node stat of the specified datanode.
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 6eb9c587b9..bc25a41795 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -505,31 +505,34 @@ public class ContainerOperationClient implements ScmClient {
}
@Override
- public void startDiskBalancer(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
- throws IOException {
- storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
- hosts);
+ public List<DatanodeAdminError> startDiskBalancer(Optional<Double> threshold,
+ Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.startDiskBalancer(threshold,
+ bandwidthInMB, parallelThread, hosts);
}
@Override
- public void stopDiskBalancer(Optional<List<String>> hosts)
+ public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.stopDiskBalancer(hosts);
+ return storageContainerLocationClient.stopDiskBalancer(hosts);
}
@Override
public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
- Optional<List<String>> hosts) throws IOException {
+ Optional<List<String>> hosts,
+ Optional<HddsProtos.DiskBalancerRunningStatus> runningStatus)
+ throws IOException {
return storageContainerLocationClient.getDiskBalancerStatus(hosts,
- ClientVersion.CURRENT_VERSION);
+ runningStatus, ClientVersion.CURRENT_VERSION);
}
@Override
- public void updateDiskBalancerConfiguration(Optional<Double> threshold,
- Optional<Double> bandwidth, Optional<List<String>> hosts)
+ public List<DatanodeAdminError> updateDiskBalancerConfiguration(
+ Optional<Double> threshold, Optional<Long> bandwidth,
+ Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
- storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
- bandwidth, hosts);
+ return storageContainerLocationClient.updateDiskBalancerConfiguration(
+ threshold, bandwidth, parallelThread, hosts);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
index 267209cf03..32719f0d83 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -54,6 +55,7 @@ public class TestDiskBalancer {
private static ScmClient storageClient;
private static MiniOzoneCluster cluster;
private static OzoneConfiguration ozoneConf;
+ private static DiskBalancerManager diskBalancerManager;
@BeforeClass
public static void setup() throws Exception {
@@ -63,6 +65,8 @@ public class TestDiskBalancer {
cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
storageClient = new ContainerOperationClient(ozoneConf);
cluster.waitForClusterToBeReady();
+ diskBalancerManager = cluster.getStorageContainerManager()
+ .getDiskBalancerManager();
for (DatanodeDetails dn: cluster.getStorageContainerManager()
.getScmNodeManager().getAllNodes()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/04: HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 95d9f87f921fc9d6e22f82c28d11c897f5286d24
Author: Symious <yi...@foxmail.com>
AuthorDate: Tue Aug 16 10:40:22 2022 +0800
HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 41 +++++
.../protocol/StorageContainerLocationProtocol.java | 30 ++++
.../scm/storage/DiskBalancerConfiguration.java | 165 +++++++++++++++++++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 55 +++++++
.../src/main/proto/ScmAdminProtocol.proto | 21 +++
.../interface-client/src/main/proto/hdds.proto | 13 ++
.../hadoop/hdds/scm/node/DiskBalancerManager.java | 176 +++++++++++++++++++++
.../hadoop/hdds/scm/node/DiskBalancerStatus.java | 50 ++++++
.../hdds/scm/node/NodeDecommissionManager.java | 123 +-------------
.../org/apache/hadoop/hdds/scm/node/NodeUtils.java | 150 ++++++++++++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 30 ++++
.../hdds/scm/server/SCMClientProtocolServer.java | 50 ++++++
.../hdds/scm/server/StorageContainerManager.java | 8 +
.../hdds/scm/node/TestDiskBalancerManager.java | 99 ++++++++++++
.../hdds/scm/node/TestNodeDecommissionManager.java | 13 +-
.../hdds/scm/cli/ContainerOperationClient.java | 36 +++++
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 96 +++++++++++
18 files changed, 1034 insertions(+), 125 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index daeb5c5ddf..76c07d1c1b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -398,4 +398,45 @@ public interface ScmClient extends Closeable {
StatusAndMessages queryUpgradeFinalizationProgress(
String upgradeClientID, boolean force, boolean readonly)
throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param count top datanodes that need balancing
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ * @param hosts If hosts is not null, return status of hosts; If hosts is
+ * null, return status of all datanodes in balancing.
+ * @return List of DatanodeDiskBalancerInfo.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index f06066b2aa..56ed8c7086 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -433,4 +433,34 @@ public interface StorageContainerLocationProtocol extends Closeable {
List<ContainerInfo> getListOfContainers(
long startContainerID, int count, HddsProtos.LifeCycleState state)
throws IOException;
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException;
+
+ /**
+ * Get DiskBalancer status.
+ */
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException;
+
+ /**
+ * Start DiskBalancer.
+ */
+ void startDiskBalancer(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Stop DiskBalancer.
+ */
+ void stopDiskBalancer(Optional<List<String>> hosts) throws IOException;
+
+ /**
+ * Update DiskBalancer Configuration.
+ */
+ void updateDiskBalancerConfiguration(
+ Optional<Double> threshold,
+ Optional<Double> bandwidth,
+ Optional<List<String>> hosts) throws IOException;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
new file mode 100644
index 0000000000..704b383679
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class contains configuration values for the DiskBalancer.
+ */
+@ConfigGroup(prefix = "hdds.datanode.disk.balancer")
+public final class DiskBalancerConfiguration {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerConfiguration.class);
+
+ @Config(key = "volume.density.threshold", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "Threshold is a percentage in the range of 0 to 100. A " +
+ "datanode is considered balanced if for each volume, the " +
+ "utilization of the volume(used space to capacity ratio) differs" +
+ " from the utilization of the datanode(used space to capacity ratio" +
+ " of the entire datanode) no more than the threshold.")
+ private double threshold = 10d;
+
+ @Config(key = "max.disk.throughputInMBPerSec", type = ConfigType.AUTO,
+ defaultValue = "10", tags = {ConfigTag.DISKBALANCER},
+ description = "The max balance speed.")
+ private double diskBandwidth = 10;
+
+ @Config(key = "parallel.thread", type = ConfigType.AUTO,
+ defaultValue = "5", tags = {ConfigTag.DISKBALANCER},
+ description = "The max parallel balance thread count.")
+ private int parallelThread = 5;
+
+ /**
+ * Gets the threshold value for DiskBalancer.
+ *
+ * @return percentage value in the range 0 to 100
+ */
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public double getThresholdAsRatio() {
+ return threshold / 100;
+ }
+
+ /**
+ * Sets the threshold value for Disk Balancer.
+ *
+ * @param threshold a percentage value in the range 0 to 100
+ */
+ public void setThreshold(double threshold) {
+ if (threshold < 0d || threshold >= 100d) {
+ throw new IllegalArgumentException(
+ "Threshold must be a percentage(double) in the range 0 to 100.");
+ }
+ this.threshold = threshold;
+ }
+
+ /**
+ * Gets the disk bandwidth value for Disk Balancer.
+ *
+ * @return max disk bandwidth per second
+ */
+ public double getDiskBandwidth() {
+ return diskBandwidth;
+ }
+
+ /**
+ * Sets the disk bandwidth value for Disk Balancer.
+ *
+ * @param diskBandwidth the bandwidth to control balance speed
+ */
+ public void setDiskBandwidth(double diskBandwidth) {
+ if (diskBandwidth <= 0d) {
+ throw new IllegalArgumentException(
+ "diskBandwidth must be a value larger than 0.");
+ }
+ this.diskBandwidth = diskBandwidth;
+ }
+
+ /**
+ * Gets the parallel thread for Disk Balancer.
+ *
+ * @return parallel thread
+ */
+ public int getParallelThread() {
+ return parallelThread;
+ }
+
+ /**
+ * Sets the parallel thread for Disk Balancer.
+ *
+ * @param parallelThread the parallel thread count
+ */
+ public void setParallelThread(int parallelThread) {
+ if (parallelThread <= 0) {
+ throw new IllegalArgumentException(
+ "parallelThread must be a value larger than 0.");
+ }
+ this.parallelThread = parallelThread;
+ }
+ @Override
+ public String toString() {
+ return String.format("Disk Balancer Configuration values:%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n" +
+ "%-50s %s%n",
+ "Key", "Value",
+ "Threshold", threshold, "Max disk bandwidth", diskBandwidth,
+ "Parallel Thread", parallelThread);
+ }
+
+ public HddsProtos.DiskBalancerConfigurationProto.Builder toProtobufBuilder() {
+ HddsProtos.DiskBalancerConfigurationProto.Builder builder =
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder();
+
+ builder.setThreshold(threshold)
+ .setDiskBandwidth(diskBandwidth)
+ .setParallelThread(parallelThread);
+ return builder;
+ }
+
+ static DiskBalancerConfiguration fromProtobuf(
+ @NotNull HddsProtos.DiskBalancerConfigurationProto proto,
+ @NotNull OzoneConfiguration ozoneConfiguration) {
+ DiskBalancerConfiguration config =
+ ozoneConfiguration.getObject(DiskBalancerConfiguration.class);
+ if (proto.hasThreshold()) {
+ config.setThreshold(proto.getThreshold());
+ }
+ if (proto.hasDiskBandwidth()) {
+ config.setDiskBandwidth(proto.getDiskBandwidth());
+ }
+ if (proto.hasParallelThread()) {
+ config.setParallelThread(proto.getParallelThread());
+ }
+ return config;
+ }
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 3728a0b1f5..0c6e98c0f9 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@ -47,5 +47,6 @@ public enum ConfigTag {
HA,
BALANCER,
UPGRADE,
- DATASTREAM
+ DATASTREAM,
+ DISKBALANCER
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index c973eb9809..0d35798352 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -42,6 +42,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
@@ -1031,6 +1034,58 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
return response.getContainerCount();
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto request =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.report)
+ .setCount(count)
+ .build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ DatanodeDiskBalancerInfoRequestProto.Builder requestBuilder =
+ DatanodeDiskBalancerInfoRequestProto.newBuilder()
+ .setInfoType(DatanodeDiskBalancerInfoType.status);
+ hosts.ifPresent(requestBuilder::addAllHosts);
+ DatanodeDiskBalancerInfoRequestProto request = requestBuilder.build();
+
+ DatanodeDiskBalancerInfoResponseProto response =
+ submitRequest(Type.DatanodeDiskBalancerInfo,
+ builder -> builder.setDatanodeDiskBalancerInfoRequest(request))
+ .getDatanodeDiskBalancerInfoResponse();
+
+ return response.getInfoList();
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ }
+
@Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index da7c6e15bd..766db3429c 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -78,6 +78,7 @@ message ScmContainerLocationRequest {
optional GetContainerReplicasRequestProto getContainerReplicasRequest = 39;
optional ReplicationManagerReportRequestProto replicationManagerReportRequest = 40;
optional ResetDeletedBlockRetryCountRequestProto resetDeletedBlockRetryCountRequest = 41;
+ optional DatanodeDiskBalancerInfoRequestProto DatanodeDiskBalancerInfoRequest = 42;
}
message ScmContainerLocationResponse {
@@ -127,6 +128,7 @@ message ScmContainerLocationResponse {
optional GetContainerReplicasResponseProto getContainerReplicasResponse = 39;
optional ReplicationManagerReportResponseProto getReplicationManagerReportResponse = 40;
optional ResetDeletedBlockRetryCountResponseProto resetDeletedBlockRetryCountResponse = 41;
+ optional DatanodeDiskBalancerInfoResponseProto DatanodeDiskBalancerInfoResponse = 42;
enum Status {
OK = 1;
@@ -175,6 +177,7 @@ enum Type {
GetReplicationManagerReport = 35;
ResetDeletedBlockRetryCount = 36;
GetClosedContainerCount = 37;
+ DatanodeDiskBalancerInfo= 38;
}
/**
@@ -326,6 +329,24 @@ message DatanodeUsageInfoResponseProto {
repeated DatanodeUsageInfoProto info = 1;
}
+enum DatanodeDiskBalancerInfoType{
+ report = 1;
+ status = 2;
+}
+
+/*
+ Datanode disk balancer status request message.
+*/
+message DatanodeDiskBalancerInfoRequestProto {
+ required DatanodeDiskBalancerInfoType infoType = 1;
+ optional uint32 count = 2;
+ repeated string hosts = 3;
+}
+
+message DatanodeDiskBalancerInfoResponseProto {
+ repeated DatanodeDiskBalancerInfoProto info = 1;
+}
+
/*
Decommission a list of hosts
*/
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 2ff39cab58..7cee2f6160 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -452,3 +452,16 @@ message ContainerBalancerConfigurationProto {
required bool shouldRun = 18;
optional int32 nextIterationIndex = 19;
}
+
+message DiskBalancerConfigurationProto {
+ optional double threshold = 1;
+ optional double diskBandwidth = 2;
+ optional int32 parallelThread = 3;
+}
+
+message DatanodeDiskBalancerInfoProto {
+ required DatanodeDetailsProto node = 1;
+ required double currentVolumeDensitySum = 2;
+ optional bool diskBalancerRunning = 3;
+ optional DiskBalancerConfigurationProto diskBalancerConf = 4;
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
new file mode 100644
index 0000000000..5ca66e9eef
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerManager.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+
+/**
+ * Maintains information about the DiskBalancer on SCM side.
+ */
+public class DiskBalancerManager {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerManager.class);
+
+ private final EventPublisher scmNodeEventPublisher;
+ private final SCMContext scmContext;
+ private final NodeManager nodeManager;
+ private Map<DatanodeDetails, DiskBalancerStatus> statusMap;
+ private boolean useHostnames;
+
+ /**
+ * Constructs DiskBalancer Manager.
+ */
+ public DiskBalancerManager(OzoneConfiguration conf,
+ EventPublisher eventPublisher,
+ SCMContext scmContext,
+ NodeManager nodeManager) {
+ this.scmNodeEventPublisher = eventPublisher;
+ this.scmContext = scmContext;
+ this.nodeManager = nodeManager;
+ this.useHostnames = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+ DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
+ this.statusMap = new ConcurrentHashMap<>();
+ }
+
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportList =
+ new ArrayList<>();
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ reportList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+
+ reportList.sort((t1, t2) -> Double.compare(t2.getCurrentVolumeDensitySum(),
+ t1.getCurrentVolumeDensitySum()));
+ return reportList.stream().limit(count).collect(Collectors.toList());
+ }
+
+ /**
+ * If hosts is not null, return status of hosts;
+ * If hosts is null, return status of all datanodes in balancing.
+ */
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusList =
+ new ArrayList<>();
+ List<DatanodeDetails> filterDns = null;
+ if (hosts.isPresent() && !hosts.get().isEmpty()) {
+ filterDns = NodeUtils.mapHostnamesToDatanodes(nodeManager, hosts.get(),
+ useHostnames);
+ }
+
+ for (DatanodeDetails datanodeDetails: nodeManager.getNodes(IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY)) {
+ if (shouldReturnDatanode(filterDns, datanodeDetails)) {
+ double volumeDensitySum =
+ getVolumeDataDensitySumForDatanodeDetails(datanodeDetails);
+ statusList.add(HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setCurrentVolumeDensitySum(volumeDensitySum)
+ .setDiskBalancerRunning(isRunning(datanodeDetails))
+ .setDiskBalancerConf(statusMap.getOrDefault(datanodeDetails,
+ DiskBalancerStatus.DUMMY_STATUS)
+ .getDiskBalancerConfiguration().toProtobufBuilder())
+ .setNode(datanodeDetails.toProto(clientVersion))
+ .build());
+ }
+ }
+ return statusList;
+ }
+
+ private boolean shouldReturnDatanode(List<DatanodeDetails> hosts,
+ DatanodeDetails datanodeDetails) {
+ if (hosts == null || hosts.isEmpty()) {
+ return isRunning(datanodeDetails);
+ } else {
+ return hosts.contains(datanodeDetails);
+ }
+ }
+
+ /**
+ * Get volume density for a specific DatanodeDetails node.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @return DiskBalancer report.
+ */
+ private double getVolumeDataDensitySumForDatanodeDetails(
+ DatanodeDetails datanodeDetails) {
+ Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo);
+
+ DatanodeInfo datanodeInfo = (DatanodeInfo) datanodeDetails;
+
+ double totalCapacity = 0d, totalUsed = 0d;
+ for (StorageContainerDatanodeProtocolProtos.StorageReportProto reportProto :
+ datanodeInfo.getStorageReports()) {
+ totalCapacity += reportProto.getCapacity();
+ totalUsed += reportProto.getScmUsed();
+ }
+
+ Preconditions.checkArgument(totalCapacity != 0);
+ double idealUsage = totalUsed / totalCapacity;
+
+ double volumeDensitySum = datanodeInfo.getStorageReports().stream()
+ .map(report ->
+ Math.abs((double)report.getScmUsed() / report.getCapacity()
+ - idealUsage))
+ .mapToDouble(Double::valueOf).sum();
+
+ return volumeDensitySum;
+ }
+
+ private boolean isRunning(DatanodeDetails datanodeDetails) {
+ return statusMap
+ .getOrDefault(datanodeDetails, DiskBalancerStatus.DUMMY_STATUS)
+ .isRunning();
+ }
+
+ @VisibleForTesting
+ public void addRunningDatanode(DatanodeDetails datanodeDetails) {
+ statusMap.put(datanodeDetails, new DiskBalancerStatus(true,
+ new DiskBalancerConfiguration()));
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
new file mode 100644
index 0000000000..ed22e80e3c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DiskBalancerStatus.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.scm.storage.DiskBalancerConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Maintains DiskBalancerConfiguration and isRunning.
+ */
+public class DiskBalancerStatus {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerStatus.class);
+
+ private boolean isRunning;
+ private DiskBalancerConfiguration diskBalancerConfiguration;
+
+ public static final DiskBalancerStatus DUMMY_STATUS =
+ new DiskBalancerStatus(false, new DiskBalancerConfiguration());
+
+ public DiskBalancerStatus(boolean isRunning, DiskBalancerConfiguration conf) {
+ this.isRunning = isRunning;
+ this.diskBalancerConfiguration = conf;
+ }
+
+ public boolean isRunning() {
+ return isRunning;
+ }
+
+ public DiskBalancerConfiguration getDiskBalancerConfiguration() {
+ return diskBalancerConfiguration;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
index a84b07d513..80f81782ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java
@@ -32,12 +32,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.util.ArrayList;
-import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -66,115 +61,6 @@ public class NodeDecommissionManager {
private static final Logger LOG =
LoggerFactory.getLogger(NodeDecommissionManager.class);
- static class HostDefinition {
- private String rawHostname;
- private String hostname;
- private int port;
-
- HostDefinition(String hostname) throws InvalidHostStringException {
- this.rawHostname = hostname;
- parseHostname();
- }
-
- public String getRawHostname() {
- return rawHostname;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public int getPort() {
- return port;
- }
-
- private void parseHostname() throws InvalidHostStringException {
- try {
- // A URI *must* have a scheme, so just create a fake one
- URI uri = new URI("empty://" + rawHostname.trim());
- this.hostname = uri.getHost();
- this.port = uri.getPort();
-
- if (this.hostname == null) {
- throw new InvalidHostStringException("The string " + rawHostname +
- " does not contain a value hostname or hostname:port definition");
- }
- } catch (URISyntaxException e) {
- throw new InvalidHostStringException(
- "Unable to parse the hoststring " + rawHostname, e);
- }
- }
- }
-
- private List<DatanodeDetails> mapHostnamesToDatanodes(List<String> hosts)
- throws InvalidHostStringException {
- List<DatanodeDetails> results = new LinkedList<>();
- for (String hostString : hosts) {
- HostDefinition host = new HostDefinition(hostString);
- InetAddress addr;
- try {
- addr = InetAddress.getByName(host.getHostname());
- } catch (UnknownHostException e) {
- throw new InvalidHostStringException("Unable to resolve host "
- + host.getRawHostname(), e);
- }
- String dnsName;
- if (useHostnames) {
- dnsName = addr.getHostName();
- } else {
- dnsName = addr.getHostAddress();
- }
- List<DatanodeDetails> found = nodeManager.getNodesByAddress(dnsName);
- if (found.size() == 0) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " (" + dnsName + ") is not running any datanodes registered"
- + " with SCM."
- + " Please check the host name.");
- } else if (found.size() == 1) {
- if (host.getPort() != -1 &&
- !validateDNPortMatch(host.getPort(), found.get(0))) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " is running a datanode registered with SCM,"
- + " but the port number doesn't match."
- + " Please check the port number.");
- }
- results.add(found.get(0));
- } else if (found.size() > 1) {
- DatanodeDetails match = null;
- for (DatanodeDetails dn : found) {
- if (validateDNPortMatch(host.getPort(), dn)) {
- match = dn;
- break;
- }
- }
- if (match == null) {
- throw new InvalidHostStringException("Host " + host.getRawHostname()
- + " is running multiple datanodes registered with SCM,"
- + " but no port numbers match."
- + " Please check the port number.");
- }
- results.add(match);
- }
- }
- return results;
- }
-
- /**
- * Check if the passed port is used by the given DatanodeDetails object. If
- * it is, return true, otherwise return false.
- * @param port Port number to check if it is used by the datanode
- * @param dn Datanode to check if it is using the given port
- * @return True if port is used by the datanode. False otherwise.
- */
- private boolean validateDNPortMatch(int port, DatanodeDetails dn) {
- for (DatanodeDetails.Port p : dn.getPorts()) {
- if (p.getValue() == port) {
- return true;
- }
- }
- return false;
- }
-
public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm,
ContainerManager containerManager, SCMContext scmContext,
EventPublisher eventQueue, ReplicationManager rm) {
@@ -225,7 +111,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> decommissionNodes(
List<String> nodes) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
@@ -290,7 +177,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> recommissionNodes(
List<String> nodes) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
@@ -327,7 +215,8 @@ public class NodeDecommissionManager {
public synchronized List<DatanodeAdminError> startMaintenanceNodes(
List<String> nodes, int endInHours) throws InvalidHostStringException {
- List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
+ List<DatanodeDetails> dns = NodeUtils.mapHostnamesToDatanodes(nodeManager,
+ nodes, useHostnames);
List<DatanodeAdminError> errors = new ArrayList<>();
for (DatanodeDetails dn : dns) {
try {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
new file mode 100644
index 0000000000..6df74d3746
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeUtils.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Util class for Node operations.
+ */
+public final class NodeUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NodeUtils.class);
+
+ private NodeUtils() {
+ }
+
+ public static List<DatanodeDetails> mapHostnamesToDatanodes(
+ NodeManager nodeManager, List<String> hosts, boolean useHostnames)
+ throws InvalidHostStringException {
+ List<DatanodeDetails> results = new LinkedList<>();
+ for (String hostString : hosts) {
+ HostDefinition host = new HostDefinition(hostString);
+ InetAddress addr;
+ try {
+ addr = InetAddress.getByName(host.getHostname());
+ } catch (UnknownHostException e) {
+ throw new InvalidHostStringException("Unable to resolve host "
+ + host.getRawHostname(), e);
+ }
+ String dnsName;
+ if (useHostnames) {
+ dnsName = addr.getHostName();
+ } else {
+ dnsName = addr.getHostAddress();
+ }
+ List<DatanodeDetails> found = nodeManager.getNodesByAddress(dnsName);
+ if (found.size() == 0) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " (" + dnsName + ") is not running any datanodes registered"
+ + " with SCM."
+ + " Please check the host name.");
+ } else if (found.size() == 1) {
+ if (host.getPort() != -1 &&
+ !validateDNPortMatch(host.getPort(), found.get(0))) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running a datanode registered with SCM,"
+ + " but the port number doesn't match."
+ + " Please check the port number.");
+ }
+ results.add(found.get(0));
+ } else if (found.size() > 1) {
+ DatanodeDetails match = null;
+ for (DatanodeDetails dn : found) {
+ if (validateDNPortMatch(host.getPort(), dn)) {
+ match = dn;
+ break;
+ }
+ }
+ if (match == null) {
+ throw new InvalidHostStringException("Host " + host.getRawHostname()
+ + " is running multiple datanodes registered with SCM,"
+ + " but no port numbers match."
+ + " Please check the port number.");
+ }
+ results.add(match);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Check if the passed port is used by the given DatanodeDetails object. If
+ * it is, return true, otherwise return false.
+ * @param port Port number to check if it is used by the datanode
+ * @param dn Datanode to check if it is using the given port
+ * @return True if port is used by the datanode. False otherwise.
+ */
+ private static boolean validateDNPortMatch(int port, DatanodeDetails dn) {
+ for (DatanodeDetails.Port p : dn.getPorts()) {
+ if (p.getValue() == port) {
+ return true;
+ }
+ }
+ return false;
+ }
+ static class HostDefinition {
+ private String rawHostname;
+ private String hostname;
+ private int port;
+
+ HostDefinition(String hostname) throws InvalidHostStringException {
+ this.rawHostname = hostname;
+ parseHostname();
+ }
+
+ public String getRawHostname() {
+ return rawHostname;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ private void parseHostname() throws InvalidHostStringException {
+ try {
+ // A URI *must* have a scheme, so just create a fake one
+ URI uri = new URI("empty://" + rawHostname.trim());
+ this.hostname = uri.getHost();
+ this.port = uri.getPort();
+
+ if (this.hostname == null) {
+ throw new InvalidHostStringException("The string " + rawHostname +
+ " does not contain a value hostname or hostname:port definition");
+ }
+ } catch (URISyntaxException e) {
+ throw new InvalidHostStringException(
+ "Unable to parse the hoststring " + rawHostname, e);
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 617d17da1d..2f50456211 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeAdminErrorResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeDiskBalancerInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
@@ -663,6 +664,14 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
getResetDeletedBlockRetryCount(
request.getResetDeletedBlockRetryCountRequest()))
.build();
+ case DatanodeDiskBalancerInfo:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeDiskBalancerInfoResponse(getDatanodeDiskBalancerInfo(
+ request.getDatanodeDiskBalancerInfoRequest(),
+ request.getVersion()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -1174,4 +1183,25 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
request.getTransactionIdList()))
.build();
}
+ public DatanodeDiskBalancerInfoResponseProto getDatanodeDiskBalancerInfo(
+ StorageContainerLocationProtocolProtos.
+ DatanodeDiskBalancerInfoRequestProto request, int clientVersion)
+ throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> infoProtoList;
+ switch (request.getInfoType()) {
+ case report:
+ infoProtoList = impl.getDiskBalancerReport(request.getCount(),
+ clientVersion);
+ break;
+ case status:
+ infoProtoList = impl.getDiskBalancerStatus(
+ Optional.of(request.getHostsList()), clientVersion);
+ break;
+ default:
+ infoProtoList = null;
+ }
+ return DatanodeDiskBalancerInfoResponseProto.newBuilder()
+ .addAllInfo(infoProtoList)
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 5b98aad4b1..e9759da960 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1126,6 +1126,56 @@ public class SCMClientProtocolServer implements
ContainerID.valueOf(startContainerID), count, state);
}
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerReport(count,
+ clientVersion);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts, int clientVersion) throws IOException {
+ // check admin authorisation
+ try {
+ getScm().checkAdminAccess(getRemoteUser());
+ } catch (IOException e) {
+ LOG.error("Authorization failed", e);
+ throw e;
+ }
+
+ return scm.getDiskBalancerManager().getDiskBalancerStatus(hosts,
+ clientVersion);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ // TODO: Send message to datanodes
+ }
+
/**
* Queries a list of Node that match a set of statuses.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index a553906969..b8727173f1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.node.DiskBalancerManager;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
@@ -230,6 +231,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private WritableContainerFactory writableContainerFactory;
private FinalizationManager finalizationManager;
private HDDSLayoutVersionManager scmLayoutVersionManager;
+ private DiskBalancerManager diskBalancerManager;
private SCMMetadataStore scmMetadataStore;
private CertificateStore certificateStore;
@@ -767,6 +769,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
.setSCMDBTransactionBuffer(scmHAManager.getDBTransactionBuffer())
.setRatisServer(scmHAManager.getRatisServer())
.build();
+ diskBalancerManager = new DiskBalancerManager(conf, eventQueue, scmContext,
+ scmNodeManager);
}
/**
@@ -1836,6 +1840,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return serviceManager;
}
+ public DiskBalancerManager getDiskBalancerManager() {
+ return diskBalancerManager;
+ }
+
/**
* Force SCM out of safe mode.
*/
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
new file mode 100644
index 0000000000..541c9764b7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDiskBalancerManager.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Unit tests for the DiskBalancer manager.
+ */
+
+public class TestDiskBalancerManager {
+
+ private DiskBalancerManager diskBalancerManager;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private String storageDir;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ storageDir = GenericTestUtils.getTempPath(
+ TestDiskBalancerManager.class.getSimpleName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ nodeManager = new MockNodeManager(true, 3);
+ diskBalancerManager = new DiskBalancerManager(conf, new EventQueue(),
+ SCMContext.emptyContext(), nodeManager);
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ diskBalancerManager.getDiskBalancerReport(2,
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(2, reportProtoList.size());
+ Assertions.assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ diskBalancerManager.addRunningDatanode(nodeManager.getAllNodes().get(0));
+
+ // Simulate users asking all status of 3 datanodes
+ List<String> dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).collect(
+ Collectors.toList());
+
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(3, statusProtoList.size());
+
+ // Simulate users asking status of 1 datanodes
+ dns = nodeManager.getAllNodes().stream().map(
+ DatanodeDetails::getIpAddress).limit(1).collect(
+ Collectors.toList());
+
+ statusProtoList =
+ diskBalancerManager.getDiskBalancerStatus(Optional.of(dns),
+ ClientVersion.CURRENT_VERSION);
+
+ Assertions.assertEquals(1, statusProtoList.size());
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
index 6851252a34..acc569392b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.node.NodeUtils.HostDefinition;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -67,26 +68,24 @@ public class TestNodeDecommissionManager {
@Test
public void testHostStringsParseCorrectly()
throws InvalidHostStringException {
- NodeDecommissionManager.HostDefinition def =
- new NodeDecommissionManager.HostDefinition("foobar");
+ HostDefinition def = new HostDefinition("foobar");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(" foobar ");
+ def = new HostDefinition(" foobar ");
assertEquals("foobar", def.getHostname());
assertEquals(-1, def.getPort());
- def = new NodeDecommissionManager.HostDefinition("foobar:1234");
+ def = new HostDefinition("foobar:1234");
assertEquals("foobar", def.getHostname());
assertEquals(1234, def.getPort());
- def = new NodeDecommissionManager.HostDefinition(
- "foobar.mycompany.com:1234");
+ def = new HostDefinition("foobar.mycompany.com:1234");
assertEquals("foobar.mycompany.com", def.getHostname());
assertEquals(1234, def.getPort());
try {
- new NodeDecommissionManager.HostDefinition("foobar:abcd");
+ new HostDefinition("foobar:abcd");
fail("InvalidHostStringException should have been thrown");
} catch (InvalidHostStringException e) {
}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 47f3d41257..6eb9c587b9 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -496,4 +496,40 @@ public class ContainerOperationClient implements ScmClient {
return storageContainerLocationClient.queryUpgradeFinalizationProgress(
upgradeClientID, force, readonly);
}
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerReport(
+ int count) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerReport(count,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void startDiskBalancer(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.startDiskBalancer(threshold, bandwidth,
+ hosts);
+ }
+
+ @Override
+ public void stopDiskBalancer(Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.stopDiskBalancer(hosts);
+ }
+
+ @Override
+ public List<HddsProtos.DatanodeDiskBalancerInfoProto> getDiskBalancerStatus(
+ Optional<List<String>> hosts) throws IOException {
+ return storageContainerLocationClient.getDiskBalancerStatus(hosts,
+ ClientVersion.CURRENT_VERSION);
+ }
+
+ @Override
+ public void updateDiskBalancerConfiguration(Optional<Double> threshold,
+ Optional<Double> bandwidth, Optional<List<String>> hosts)
+ throws IOException {
+ storageContainerLocationClient.updateDiskBalancerConfiguration(threshold,
+ bandwidth, hosts);
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
new file mode 100644
index 0000000000..267209cf03
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class tests disk balancer operations.
+ */
+public class TestDiskBalancer {
+
+ /**
+ * Set a timeout for each test.
+ */
+ @Rule
+ public Timeout timeout = Timeout.seconds(300);
+
+ private static ScmClient storageClient;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration ozoneConf;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ ozoneConf = new OzoneConfiguration();
+ ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
+ cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
+ storageClient = new ContainerOperationClient(ozoneConf);
+ cluster.waitForClusterToBeReady();
+
+ for (DatanodeDetails dn: cluster.getStorageContainerManager()
+ .getScmNodeManager().getAllNodes()) {
+ ((DatanodeInfo) dn).updateStorageReports(
+ HddsTestUtils.getRandomNodeReport(3, 1).getStorageReportList());
+ }
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerReport() throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> reportProtoList =
+ storageClient.getDiskBalancerReport(2);
+
+ assertEquals(2, reportProtoList.size());
+ Assert.assertTrue(
+ reportProtoList.get(0).getCurrentVolumeDensitySum()
+ >= reportProtoList.get(1).getCurrentVolumeDensitySum());
+ }
+
+ @Test
+ public void testDatanodeDiskBalancerStatus() throws IOException {
+ // TODO: Test status command with datanodes in balancing
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 04/04: HDDS-7234. Add a common option for DiskBalancer commands (#3762)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 6ae2da8218f2ba54cd5c378884c9354ff3e25389
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 22 10:00:39 2022 +0800
HDDS-7234. Add a common option for DiskBalancer commands (#3762)
---
.../scm/cli/datanode/DiskBalancerCommands.java | 18 ++---
.../cli/datanode/DiskBalancerCommonOptions.java | 84 ++++++++++++++++++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 27 +++----
.../cli/datanode/DiskBalancerStatusSubcommand.java | 7 +-
.../cli/datanode/DiskBalancerStopSubcommand.java | 29 +++-----
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 27 +++----
6 files changed, 123 insertions(+), 69 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
index 649ee59eaa..dd70bbb491 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -38,36 +38,38 @@ import java.util.concurrent.Callable;
* [ -t/--threshold {@literal <threshold>}]
* [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
* [ -p/--parallelThread {@literal <parallelThread>}]
- * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ -a/--all {@literal <alldatanodes>}]
+ * [ -d/--datanodes {@literal <datanodes>}]
* [ {@literal <hosts>}]
* Examples:
- * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -d {@literal <hosts>}
* start balancer with default values in the configuration on specified
* datanodes
* ozone admin datanode diskbalancer start -a
* start balancer with default values in the configuration on all
* datanodes in the cluster
- * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -t 5 -d {@literal <hosts>}
* start balancer with a threshold of 5%
- * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -b 20 -d {@literal <hosts>}
* start balancer with maximum 20MB/s diskbandwidth
- * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * ozone admin datanode diskbalancer start -p 5 -d {@literal <hosts>}
* start balancer with 5 parallel thread on each datanode
* To stop:
* ozone admin datanode diskbalancer stop -a
* stop diskblancer on all datanodes
- * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * ozone admin datanode diskbalancer stop -d {@literal <hosts>};
* stop diskblancer on all datanodes
* To update:
* ozone admin datanode diskbalancer update -a
* update diskblancer configuration on all datanodes
- * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * ozone admin datanode diskbalancer update -d {@literal <hosts>};
* update diskblancer configuration on all datanodes
* To get report:
* ozone admin datanode diskbalancer report -c 10
* retrieve at most 10 datanodes that needs diskbalance most
* To get status:
- * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * ozone admin datanode diskbalancer status -s RUNNING -d
+ * {@literal <hosts>}
* return the diskbalancer status on datanodes where diskbalancer are in
* Running state
*
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
new file mode 100644
index 0000000000..744de5169f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import picocli.CommandLine;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Common options for DiskBalancer commands.
+ */
+public class DiskBalancerCommonOptions {
+
+ @CommandLine.Option(names = {"-a", "--all"},
+ description = "Run commands on all datanodes.")
+ private boolean allHosts;
+
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Run commands on specific datanodes, the content can be " +
+ "a list of hostnames or IPs. " +
+ "Examples: hostname1,hostname2,hostname3 or ip1,ip2,ip3")
+ private List<String> datanodes = new ArrayList<>();
+
+ /**
+ * Check the common options of DiskBalancerCommand.
+ * @return if the check passed
+ */
+ public boolean check() {
+ if (datanodes.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify at least " +
+ "one datanode or use \"-a(--all)\" to start diskBalancer " +
+ "on all datanodes");
+ return false;
+ }
+ if (datanodes.size() != 0 && allHosts) {
+ System.out.println("Invalid option selection. " +
+ "Use either \"-a(--all)\" or \"-d(--datanodes)\".");
+ return false;
+ }
+ return true;
+ }
+
+ public String getHostString() {
+ return isAllHosts() ? "All datanodes" : String.join("\n", getDatanodes());
+ }
+
+ public Optional<List<String>> getSpecifiedDatanodes() {
+ return getDatanodes().size() == 0 ?
+ Optional.empty() : Optional.of(getDatanodes());
+ }
+
+ public boolean isAllHosts() {
+ return allHosts;
+ }
+
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+
+ public List<String> getDatanodes() {
+ return datanodes;
+ }
+
+ public void setDatanodes(List<String> datanodes) {
+ this.datanodes = datanodes;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
index e3862ba23f..f58c7a1f84 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified. Please specify " +
- "\"--allDatanodes\" to start diskBalancer on all datanodes");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Start DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -95,6 +84,6 @@ public class DiskBalancerStartSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
index 6a61876479..84dd3abe95 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -21,9 +21,9 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
import java.util.ArrayList;
@@ -48,10 +48,11 @@ public class DiskBalancerStatusSubcommand extends ScmSubcommand {
new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
@Option(names = {"-s", "--state"},
- description = "RUNNING, STOPPED, UNKNOWN")
+ description = "RUNNING, STOPPED, UNKNOWN. Default state is RUNNING.")
private String state = "RUNNING";
- @Parameters(description = "List of fully qualified host names")
+ @CommandLine.Option(names = {"-d", "--datanodes"},
+ description = "Get diskBalancer status on specific datanodes.")
private List<String> hosts = new ArrayList<>();
@Override
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
index 9479c030a6..f63f880a44 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -22,14 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
/**
* Handler to stop disk balancer.
@@ -40,29 +37,21 @@ import java.util.Optional;
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class DiskBalancerStopSubcommand extends ScmSubcommand {
- @Option(names = {"-a", "--allDatanodes"},
- description = "Stop diskBalancer on all datanodes.")
- private boolean allHosts;
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
+ if (!commonOptions.check()) {
return;
}
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
- return;
- }
- List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
- Optional.empty() : Optional.of(hosts));
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(
+ commonOptions.getSpecifiedDatanodes());
System.out.println("Stopping DiskBalancer on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
System.err.println("Error: " + error.getHostname() + ": "
@@ -77,6 +66,6 @@ public class DiskBalancerStopSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
index b19fb6a53a..28ed4fd325 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -22,12 +22,11 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -55,31 +54,21 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
description = "Max parallelThread for DiskBalancer.")
private Optional<Integer> parallelThread;
- @Option(names = {"-a", "--allDatanodes"},
- description = "Start diskBalancer on all datanodes.")
- private boolean allHosts;
-
- @Parameters(description = "List of fully qualified host names")
- private List<String> hosts = new ArrayList<>();
+ @CommandLine.Mixin
+ private DiskBalancerCommonOptions commonOptions =
+ new DiskBalancerCommonOptions();
@Override
public void execute(ScmClient scmClient) throws IOException {
- if (hosts.size() == 0 && !allHosts) {
- System.out.println("Datanode not specified.");
- return;
- }
- if (hosts.size() != 0 && allHosts) {
- System.out.println("Confused options. Omit \"--allDatanodes\" or " +
- "Datanodes.");
+ if (!commonOptions.check()) {
return;
}
List<DatanodeAdminError> errors =
scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
- parallelThread,
- hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+ parallelThread, commonOptions.getSpecifiedDatanodes());
System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
- (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ commonOptions.getHostString());
if (errors.size() > 0) {
for (DatanodeAdminError error : errors) {
@@ -93,6 +82,6 @@ public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
@VisibleForTesting
public void setAllHosts(boolean allHosts) {
- this.allHosts = allHosts;
+ this.commonOptions.setAllHosts(allHosts);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 03/04: HDDS-7205. DiskBalancer CLI (#3739)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 28cfeab74b184830e77add701b2f4ab3ae81b86b
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Sep 15 10:02:49 2022 +0800
HDDS-7205. DiskBalancer CLI (#3739)
---
.../hdds/scm/server/SCMClientProtocolServer.java | 6 +-
.../hdds/scm/cli/datanode/DatanodeCommands.java | 3 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 99 ++++++++
.../cli/datanode/DiskBalancerReportSubcommand.java | 70 ++++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 100 ++++++++
.../cli/datanode/DiskBalancerStatusSubcommand.java | 103 ++++++++
.../cli/datanode/DiskBalancerStopSubcommand.java | 82 +++++++
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 98 ++++++++
.../cli/datanode/TestDiskBalancerSubCommand.java | 259 +++++++++++++++++++++
9 files changed, 816 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index d3f23089be..a20edaa05a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1162,14 +1162,14 @@ public class SCMClientProtocolServer implements
Optional<Long> bandwidthInMB, Optional<Integer> parallelThread,
Optional<List<String>> hosts) throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@Override
public List<DatanodeAdminError> stopDiskBalancer(Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
@@ -1179,7 +1179,7 @@ public class SCMClientProtocolServer implements
Optional<Integer> parallelThread, Optional<List<String>> hosts)
throws IOException {
// TODO: Send message to datanodes
- return null;
+ return new ArrayList<DatanodeAdminError>();
}
/**
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
index 8cb2114f57..4c7f71037f 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
@@ -41,7 +41,8 @@ import java.util.concurrent.Callable;
DecommissionSubCommand.class,
MaintenanceSubCommand.class,
RecommissionSubCommand.class,
- UsageInfoSubcommand.class
+ UsageInfoSubcommand.class,
+ DiskBalancerCommands.class
})
@MetaInfServices(SubcommandWithParent.class)
public class DatanodeCommands implements Callable<Void>, SubcommandWithParent {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
new file mode 100644
index 0000000000..649ee59eaa
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand to group disk balancer related operations.
+ *
+ * <p>The balancer is a tool that balances space usage on an Ozone datanode
+ * when some disks become full or when new empty disks were added to a datanode.
+ *
+ * <p>SYNOPSIS
+ * <pre>
+ * To start:
+ * ozone admin datanode diskbalancer start
+ * [ -t/--threshold {@literal <threshold>}]
+ * [ -b/--bandwidthInMB {@literal <bandwidthInMB>}]
+ * [ -p/--parallelThread {@literal <parallelThread>}]
+ * [ -a/--alldatanodes {@literal <alldatanodes>}]
+ * [ {@literal <hosts>}]
+ * Examples:
+ * ozone admin datanode diskbalancer start {@literal <hosts>}
+ * start balancer with default values in the configuration on specified
+ * datanodes
+ * ozone admin datanode diskbalancer start -a
+ * start balancer with default values in the configuration on all
+ * datanodes in the cluster
+ * ozone admin datanode diskbalancer start -t 5 {@literal <hosts>}
+ * start balancer with a threshold of 5%
+ * ozone admin datanode diskbalancer start -b 20 {@literal <hosts>}
+ * start balancer with maximum 20MB/s diskbandwidth
+ * ozone admin datanode diskbalancer start -p 5 {@literal <hosts>}
+ * start balancer with 5 parallel thread on each datanode
+ * To stop:
+ * ozone admin datanode diskbalancer stop -a
+ * stop diskblancer on all datanodes
+ * ozone admin datanode diskbalancer stop {@literal <hosts>};
+ * stop diskblancer on all datanodes
+ * To update:
+ * ozone admin datanode diskbalancer update -a
+ * update diskblancer configuration on all datanodes
+ * ozone admin datanode diskbalancer update {@literal <hosts>};
+ * update diskblancer configuration on all datanodes
+ * To get report:
+ * ozone admin datanode diskbalancer report -c 10
+ * retrieve at most 10 datanodes that needs diskbalance most
+ * To get status:
+ * ozone admin datanode diskbalancer status -s RUNNING {@literal <hosts>}
+ * return the diskbalancer status on datanodes where diskbalancer are in
+ * Running state
+ *
+ * </pre>
+ */
+
+@Command(
+ name = "diskbalancer",
+ description = "DiskBalancer specific operations",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ DiskBalancerStartSubcommand.class,
+ DiskBalancerStopSubcommand.class,
+ DiskBalancerUpdateSubcommand.class,
+ DiskBalancerReportSubcommand.class,
+ DiskBalancerStatusSubcommand.class
+ })
+public class DiskBalancerCommands implements Callable<Void> {
+
+ @Spec
+ private CommandSpec spec;
+
+ @Override
+ public Void call() throws Exception {
+ GenericCli.missingSubcommand(spec);
+ return null;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
new file mode 100644
index 0000000000..deb7b07043
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Handler to get Datanode Volume Density report.
+ */
+@Command(
+ name = "report",
+ description = "Get Datanode Volume Density Report",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerReportSubcommand extends ScmSubcommand {
+ @Option(names = {"-c", "--count"},
+ description = "Result count to return. Sort by Volume Density " +
+ "in descending order.")
+ private int count;
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerReport(count);
+ System.out.println(generateReport(resultProto));
+ }
+
+ private String generateReport(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Report result:%n" +
+ "%-50s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
new file mode 100644
index 0000000000..e3862ba23f
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to start disk balancer.
+ */
+@Command(
+ name = "start",
+ description = "Start DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStartSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified. Please specify " +
+ "\"--allDatanodes\" to start diskBalancer on all datanodes");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.startDiskBalancer(threshold, bandwidthInMB, parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Start DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not start DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
new file mode 100644
index 0000000000..6a61876479
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Handler to get disk balancer status.
+ */
+@Command(
+ name = "status",
+ description = "Get Datanode DiskBalancer Status",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStatusSubcommand extends ScmSubcommand {
+
+ private final Set<String> stateSet =
+ new HashSet<>(Arrays.asList("RUNNING", "STOPPED", "UNKNOWN"));
+
+ @Option(names = {"-s", "--state"},
+ description = "RUNNING, STOPPED, UNKNOWN")
+ private String state = "RUNNING";
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (state != null && !stateSet.contains(state.toUpperCase(Locale.ROOT))) {
+ System.err.println("Unsupported state: " + state);
+ }
+
+ assert state != null;
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> resultProto =
+ scmClient.getDiskBalancerStatus(
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts),
+ Optional.of(HddsProtos.DiskBalancerRunningStatus.valueOf(
+ state.toUpperCase(Locale.ROOT)))
+ );
+
+ System.out.println(generateStatus(resultProto));
+ }
+
+ private String generateStatus(
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> protos) {
+ StringBuilder formatBuilder = new StringBuilder("Status result:%n" +
+ "%-50s %s %s %s %s %s%n");
+
+ List<String> contentList = new ArrayList<>();
+ contentList.add("Datanode");
+ contentList.add("VolumeDensity");
+ contentList.add("Status");
+ contentList.add("Threshold");
+ contentList.add("BandwidthInMB");
+ contentList.add("ParallelThread");
+
+ for (HddsProtos.DatanodeDiskBalancerInfoProto proto: protos) {
+ formatBuilder.append("%-50s %s %s %s %s %s%n");
+ contentList.add(proto.getNode().getHostName());
+ contentList.add(String.valueOf(proto.getCurrentVolumeDensitySum()));
+ contentList.add(proto.getRunningStatus().name());
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getThreshold()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getDiskBandwidthInMB()));
+ contentList.add(
+ String.valueOf(proto.getDiskBalancerConf().getParallelThread()));
+ }
+
+ return String.format(formatBuilder.toString(),
+ contentList.toArray(new String[0]));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
new file mode 100644
index 0000000000..9479c030a6
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to stop disk balancer.
+ */
+@Command(
+ name = "stop",
+ description = "Stop DiskBalancer",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerStopSubcommand extends ScmSubcommand {
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Stop diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors = scmClient.stopDiskBalancer(allHosts ?
+ Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Stopping DiskBalancer on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ // Throwing the exception will cause a non-zero exit status for the
+ // command.
+ throw new IOException(
+ "Some nodes could not stop DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
new file mode 100644
index 0000000000..b19fb6a53a
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Handler to update disk balancer configuration.
+ */
+@Command(
+ name = "update",
+ description = "Update DiskBalancer Configuration",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class DiskBalancerUpdateSubcommand extends ScmSubcommand {
+
+ @Option(names = {"-t", "--threshold"},
+ description = "Percentage deviation from average utilization of " +
+ "the disks after which a datanode will be rebalanced (for " +
+ "example, '10' for 10%%).")
+ private Optional<Double> threshold;
+
+ @Option(names = {"-b", "--bandwidthInMB"},
+ description = "Maximum bandwidth for DiskBalancer per second.")
+ private Optional<Long> bandwidthInMB;
+
+ @Option(names = {"-p", "--parallelThread"},
+ description = "Max parallelThread for DiskBalancer.")
+ private Optional<Integer> parallelThread;
+
+ @Option(names = {"-a", "--allDatanodes"},
+ description = "Start diskBalancer on all datanodes.")
+ private boolean allHosts;
+
+ @Parameters(description = "List of fully qualified host names")
+ private List<String> hosts = new ArrayList<>();
+
+ @Override
+ public void execute(ScmClient scmClient) throws IOException {
+ if (hosts.size() == 0 && !allHosts) {
+ System.out.println("Datanode not specified.");
+ return;
+ }
+ if (hosts.size() != 0 && allHosts) {
+ System.out.println("Confused options. Omit \"--allDatanodes\" or " +
+ "Datanodes.");
+ return;
+ }
+ List<DatanodeAdminError> errors =
+ scmClient.updateDiskBalancerConfiguration(threshold, bandwidthInMB,
+ parallelThread,
+ hosts.size() == 0 ? Optional.empty() : Optional.of(hosts));
+
+ System.out.println("Update DiskBalancer Configuration on datanode(s):\n" +
+ (allHosts ? "All datanodes" : String.join("\n", hosts)));
+
+ if (errors.size() > 0) {
+ for (DatanodeAdminError error : errors) {
+ System.err.println("Error: " + error.getHostname() + ": "
+ + error.getError());
+ }
+ throw new IOException(
+ "Some nodes could not update DiskBalancer.");
+ }
+ }
+
+ @VisibleForTesting
+ public void setAllHosts(boolean allHosts) {
+ this.allHosts = allHosts;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
new file mode 100644
index 0000000000..702e6ee7c5
--- /dev/null
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommand.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests to validate the DiskBalancerSubCommand class includes the
+ * correct output when executed against a mock client.
+ */
+public class TestDiskBalancerSubCommand {
+
+ private DiskBalancerStopSubcommand stopCmd;
+ private DiskBalancerStartSubcommand startCmd;
+ private DiskBalancerUpdateSubcommand updateCmd;
+ private DiskBalancerReportSubcommand reportCmd;
+ private DiskBalancerStatusSubcommand statusCmd;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ private Random random = new Random();
+
+ @BeforeEach
+ public void setup() throws UnsupportedEncodingException {
+ stopCmd = new DiskBalancerStopSubcommand();
+ startCmd = new DiskBalancerStartSubcommand();
+ updateCmd = new DiskBalancerUpdateSubcommand();
+ reportCmd = new DiskBalancerReportSubcommand();
+ statusCmd = new DiskBalancerStatusSubcommand();
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testDiskBalancerReportSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test report
+ Mockito.when(scmClient.getDiskBalancerReport(Mockito.any(Integer.class)))
+ .thenReturn(generateReport(10));
+
+ reportCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStatusSubcommand()
+ throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+
+ //test status
+ Mockito.when(scmClient.getDiskBalancerStatus(Mockito.any(), Mockito.any()))
+ .thenReturn(generateStatus(10));
+
+ statusCmd.execute(scmClient);
+
+ // 2 Headers + 10 results
+ assertEquals(12, newLineCount(outContent.toString(DEFAULT_ENCODING)));
+ }
+
+ @Test
+ public void testDiskBalancerStartSubcommand() throws IOException {
+ startCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not start DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.startDiskBalancer(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ startCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ startCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerUpdateSubcommand() throws IOException {
+ updateCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not update DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.updateDiskBalancerConfiguration(Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ updateCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ updateCmd.setAllHosts(false);
+ }
+
+ @Test
+ public void testDiskBalancerStopSubcommand() throws IOException {
+ stopCmd.setAllHosts(true);
+ ScmClient scmClient = mock(ScmClient.class);
+
+ // Return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(10));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ assertEquals("Some nodes could not stop DiskBalancer.", e.getMessage());
+ }
+
+ // Do not return error
+ Mockito.when(scmClient.stopDiskBalancer(Mockito.any()))
+ .thenReturn(generateError(0));
+
+ try {
+ stopCmd.execute(scmClient);
+ } catch (IOException e) {
+ fail("Should not catch exception here.");
+ }
+
+ stopCmd.setAllHosts(false);
+ }
+
+
+ private List<DatanodeAdminError> generateError(int count) {
+ List<DatanodeAdminError> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(new DatanodeAdminError(UUID.randomUUID().toString(),
+ "ERROR"));
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateReport(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateReport());
+ }
+ return result;
+ }
+
+ private List<HddsProtos.DatanodeDiskBalancerInfoProto> generateStatus(
+ int count) {
+ List<HddsProtos.DatanodeDiskBalancerInfoProto> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ result.add(generateStatus());
+ }
+ return result;
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateReport() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDiskBalancerInfoProto generateStatus() {
+ return HddsProtos.DatanodeDiskBalancerInfoProto.newBuilder()
+ .setNode(generateDatanodeDetails())
+ .setCurrentVolumeDensitySum(random.nextDouble())
+ .setRunningStatus(HddsProtos.DiskBalancerRunningStatus.
+ valueOf(random.nextInt(2) + 1))
+ .setDiskBalancerConf(
+ HddsProtos.DiskBalancerConfigurationProto.newBuilder().build())
+ .build();
+ }
+
+ private HddsProtos.DatanodeDetailsProto generateDatanodeDetails() {
+ return HddsProtos.DatanodeDetailsProto.newBuilder()
+ .setHostName(UUID.randomUUID().toString())
+ .setIpAddress("1.1.1.1")
+ .build();
+ }
+
+ private int newLineCount(String str) {
+ int res = 0;
+ String[] lines = str.split("\n");
+ for (String line : lines) {
+ if (line.length() != 0) {
+ res++;
+ }
+ }
+ return res;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org