You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by as...@apache.org on 2024/12/19 23:28:23 UTC
(ozone) 02/02: Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation-merge
This is an automated email from the ASF dual-hosted git repository.
aswinshakil pushed a commit to branch HDDS-10239-container-reconciliation
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit cfcabf14223f380b22a483c4ca5a6a38673bd1fc
Merge: 7bcc943077 0066526dce
Author: Aswin Shakil Balasubramanian <as...@gmail.com>
AuthorDate: Thu Dec 19 12:11:01 2024 -0800
Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation-merge
Commits:
0066526dce HDDS-11869. Enable OM Ratis in TestOzoneDelegationTokenSecretManager (#7594)
4fe166d9dc HDDS-11957. Make breadcrumb scrollable for long path names in DiskUsage page (#7590)
a523e17dac HDDS-11846. [Recon] Recon Schema version_number column is always set as -1. (#7554)
f5ff2f05a5 HDDS-11868. Enable OM Ratis in TestQuotaRepairTask (#7593)
3a0e3b5364 HDDS-11845. Extract k8s definitions for HttpFS and Recon from getting-started example (#7523)
6e0c753d78 HDDS-11509. logging improvements for directory deletion (#7314)
1f29e05ea1 HDDS-11934. Split compat suite to old/new (#7578)
bde8cf444d HDDS-11759. Remove LegacyReplicationManager (#7580)
a27e4ec1ac HDDS-11779. Add DN metrics to show deletion progress (#7552)
976e45f334 HDDS-11711. Add SCM metrics for delete commands sent and response received per datanode (#7522)
c28e16ec68 HDDS-11950. Enable sortpom in dev-support module. (#7586)
dae388b7ee HDDS-11907. OzoneSecretKey does not need to implement Writable (#7574)
8bb0587d37 HDDS-11712. Process other DeletedBlocksTransaction before retrying failed one. (#7532)
3648b592a9 HDDS-11906. Add sortpom dependency, sort root POM. (#7555)
54f0272468 HDDS-11807. Make callId different for each request in openKeyCleanupService (#7551)
c523825f02 HDDS-11926 - Rename bucket name for bucket info/ls for linked buckets (#7581)
daf2f9fce0 HDDS-11863. Speed up TestFSORepairTool (#7561)
f5e5493d01 HDDS-11927. Fix flaky TestContainerBalancerStatusInfo.testGetCurrentStatisticsWhileBalancingInProgress (#7579)
bef2415dbb HDDS-11940. Bump jline to 3.28.0 (#7576)
1453fd922b HDDS-11935. Bump develocity-maven-extension to 1.23 (#7577)
202b0c731b HDDS-11860. Improve BufferUtils.writeFully. (#7564)
008f9a61f2 HDDS-11852. Reduce duplication in some GenericCli subclasses (#7553)
7a460803f5 HDDS-11914. Snapshot diff should not filter SST Files based by reading SST file reader (#7563)
1835326b16 HDDS-11927. Mark testGetCurrentStatisticsWhileBalancingInProgress as flaky
66ccc25280 HDDS-11908. Snapshot diff DAG traversal should not skip node based on prefix presence (#7567)
16ba289fe3 Revert "HDDS-11413. PipelineManagerImpl lock optimization reduces AllocateBlock latency (#7160)"
bf6f323109 HDDS-11413. PipelineManagerImpl lock optimization reduces AllocateBlock latency (#7160)
853d6572a8 HDDS-11893. Fix full snapshot diff fallback logic because of DAG pruning (#7549)
b5d04e2086 HDDS-11915. Netty OpenSsl not available in acceptance tests on arm64 (#7570)
85364907d6 HDDS-11367. Fix flaky balancer robot test (#7569)
745ed1cd23 HDDS-11367. Improve ozone balancing status command output (#7139)
6b9cbe0e47 HDDS-11909. Intermittent timeout building Hadoop in s3a test (#7559)
eea5600871 HDDS-11911. Return consistent error code when snapshot is not found in the DB or Snapshot Chain. (#7557)
e8f3b25241 HDDS-11873. Skip old-only xcompat read tests (#7534)
ec348a7e48 HDDS-11889. Include Maven dependencies for hdds-rocks-native in cache (#7546)
aa37ae8a8e HDDS-11885. Download Hadoop for S3A test from mirrors if available (#7545)
befd64e068 HDDS-11694. Safemode Improvement: Introduce factory class to create safemode rules. (#7433)
a46153d221 HDDS-11872. Disable Apache snapshots repo (#7536)
092b00087d HDDS-11890. Update project description in GitHub (#7547)
80c6446a11 HDDS-8101. Add tool to repair broken FSO tree (#7368)
23197e26ac HDDS-11605. Directory deletion service should support multiple threads (#7349)
055b13c8dd HDDS-11751. Use Java 21 in CI (#7458)
d0d82c5112 HDDS-11886. Bump license-maven-plugin to 2.5.0 (#7539)
f7fe30a165 HDDS-11691. Support object tags in ObjectEndpointStreaming#put (#7543)
9854591537 HDDS-11882. Make BOM, not aggregate one (#7544)
af345b23df HDDS-11877. Enable Maven cache for more checks (#7538)
51c6ed65ad HDDS-11830. Subcommands should not extend GenericCli .(#7537)
959a39d4d7 HDDS-11851. Finer-grained subcommand interface for OzoneDebug and OzoneRepair. (#7526)
d4c41e542e HDDS-11334. Improve EC xcompat acceptance test (#7492)
8526f2e81f HDDS-11826. Interactive mode for ozone shell. (#7515)
fc6371039b HDDS-11833. Return NotImplemented for S3 put-object-acl request. (#7531)
e8ad7ad27a HDDS-11782. ozone debug ldb --with-keys defaults to false instead of true (#7521)
cb0a402bea HDDS-11859. Remove mention of fuse from s3 interface docs page (#7530)
e17f92c4fc HDDS-10821. Ensure ChunkBuffer fully writes buffer to FileChannel (#6652)
d66c0888fb HDDS-11848. Serialization bug in Recon listKeys API (#7524)
faab1e8e5f HDDS-11656. Default native ACL limits to user and user's primary group (#7455)
8a1967ea06 HDDS-11719. Remove dependency on server components from ozonefs-common (#7438)
9fcecc194a HDDS-11855. Mark TestContainerBalancerDatanodeNodeLimit#checkIterationResultException as flaky
65df3080e3 HDDS-11794. Display HostName in OM / SCM Overview. (#7482)
0bde3a293f HDDS-11410. Refactoring more tests from TestContainerBalancerTask (#7156)
98e070e6ee HDDS-11728. Refactor subcommand layouts of ozone debug and repair (#7489)
1c5676bcf9 HDDS-11849. Mark TestBlockOutputStreamWithFailures.test2DatanodesFailure as flaky
9dd8bb9105 HDDS-11847. Mark TestSnapshotDeletingServiceIntegrationTest#testParallelExcecutionOfKeyDeletionAndSnapshotDeletion as flaky
b27714f572 HDDS-11266. Update proto.lock for Ozone 1.4.1 (#7504)
9b619373b6 HDDS-11806. Add HttpFS and Recon in getting-started k8s example (#7485)
77ce962f34 HDDS-10568. When the ldb command is executed, it is output by line (#7467)
69538b0b85 HDDS-11687. Robot warning: replace "is not" with "!=" (#7516)
b60d897c90 HDDS-11820. Create test principals at test run time (#7507)
6ba309a359 HDDS-11831. Finer-grained interface for dynamically registered subcommands (#7514)
db36e39ad0 HDDS-11822. Register subcommands in OzoneShell (#7513)
e6bd3f5c52 HDDS-11829. Bump zstd-jni to 1.5.6-8 (#7510)
09a4a904d2 HDDS-11827. Bump exec-maven-plugin to 3.5.0 (#7512)
ad867bcef6 HDDS-11824. Bump sqlite-jdbc to 3.47.1.0 (#7511)
850306def3 HDDS-11823. Bump cyclonedx-maven-plugin to 2.9.1 (#7508)
34f9d9e870 HDDS-11742. Update metrics with leaderId if known when starting SCM (#7471)
2c6c1169b6 HDDS-11265. Add Ozone 1.4.1 to compatibility acceptance tests (#7503)
ebcdc6a1ef HDDS-11810. Secure acceptance test on arm64 fails with LoginException: Checksum failed (#7498)
7f40624ebf HDDS-11821. Mark TestECKeyOutputStream#testECKeyCreatetWithDatanodeIdChange as unhealthy
9b261560d6 HDDS-11811. rocksdbjni deleted on exit could be used by other components #7493
3f92663aed HDDS-11785. DataNode aborts ContainerStateMachine if it does not know any follower next index (#7480)
f0a2c87f30 HDDS-11773. Prevent frequent DataNode Ratis snapshotting. (#7473)
1383c18bbe HDDS-11718. Some CI checks passing despite error (#7483)
f98eac2099 HDDS-11561. Refactor Open Key Search Endpoint and Consolidate with OmDBInsightEndpoint Using StartPrefix Parameter. (#7336)
a99ab2706e HDDS-11243. SCM SafeModeRule Support EC. (#7008)
6871547273 HDDS-11716. Address Incomplete Upgrade Scenario in Recon Upgrade Framework (#7452)
9bc9145d48 HDDS-10411. Support incremental ChunkBuffer checksum calculation (#7189)
579a38e6c4 HDDS-11723. Tool to better micro benchmark hbase performance in Ozone (#7463)
cc1a37476b HDDS-11704. Hadoop test leaves running containers in case of failure (#7435)
c4b2056e6f Update documentation to mention that container schemaV3 is default (#7481)
20c4cfa349 HDDS-11386. Multithreading bug in ContainerBalancerTask (#7339)
b090312574 HDDS-11780. Increase client write retry when SCM is in safe mode (#7470)
f8e4db926f HDDS-11793. Bump maven-checkstyle-plugin to 3.6.0 (#7476)
6a1ff845fd HDDS-11791. Bump commons-io to 2.18.0 (#7478)
d7f8235fde HDDS-11788. Bump log4j2 to 2.24.2 (#7479)
6547de7217 HDDS-11790. Bump commons-lang3 to 3.17.0 (#7475)
1d8abd6989 HDDS-11789. Bump zstd-jni to 1.5.6-7 (#7477)
6ca7230b49 HDDS-11769. Add tools folder into ozone src package. (#7466)
3b8ed58c6a HDDS-11682. Bump maven-resources-plugin to 3.3.0 (#7384)
f4a9ee0c2f HDDS-11702. Merge test_bucket_encryption into robot compatibility test (#7451)
d6a54880fd HDDS-11713. Use seek to reach the start transaction. (#7460)
d52615ae54 HDDS-11733. Remove okio versioning and unused okhttp dependency (#7447)
1a49991372 HDDS-11617. Update hadoop to 3.4.1 (#7376)
9945de6c75 HDDS-11667. Validating DatanodeID on any request to the datanode (#7418)
fc6a2ea144 HDDS-11650. ContainerId list to track all containers created in a datanode (#7402)
a8db9cd476 HDDS-11749. Extract moveToTrash implementation to client code (#7453)
3ba347484b HDDS-11755. mktemp --suffix does not work on Mac (#7457)
433c7bb186 HDDS-11729. Update skipRecon property to skip only frontend build (#7454)
6b40003bb2 HDDS-11739. Extract generic unmarshaller for S3 requests (#7449)
c7f65e7ebb HDDS-11740. Add debug command to show internal component versions (#7450)
0f7104eec0 HDDS-11708. Recon ListKeys API should return a proper http response status code if NSSummary rebuild is in progress. (#7437)
0e0d5e9bad HDDS-11163. Improve Heatmap page UI (#7420)
2cef3935e0 HDDS-11696. Limit max number of entries in list keys/status response (#7431)
e96e314b28 HDDS-11697. Integrate Ozone Filesystem Implementation with Ozone ListStatusLight API (#7440)
ebcbce78e4 HDDS-11644. Close OMLayoutVersionManager (#7445)
20e4969679 HDDS-11737. UnsupportedOperationException in S3 setBucketAcl (#7448)
b25218172d HDDS-10804. Include only limited set of ports in Pipeline proto (#6655)
79ca95672a HDDS-8829. Symmetric Keys for Delegation Tokens (#7394)
3e798e6ce5 HDDS-11698. Use hadoop images from GitHub in CI (#7432)
3e278b7609 HDDS-10655. Support PutObjectTagging, GetObjectTagging, and DeleteObjectTagging (#6756)
036e727327 HDDS-11732. Fix ACL check on bucket resolution while reading from snapshot (#7446)
dbda703281 HDDS-11736. Bump maven-javadoc-plugin to 3.11.1 (#7444)
238f232420 HDDS-11692. Skip spotbugs for modules with only generated code. (#7428)
f60ad6101e HDDS-11705. Snapshot operations on linked buckets should work on actual underlying bucket (#7434)
dd22dbef89 HDDS-11615. Add Upgrade Action for Initial Schema Constraints for Unhealthy Container Table in Recon. (#7372)
4066c7c2e4 HDDS-117. Add convenience methods for port management in DatanodeDetails (#7408)
12419fae1f HDDS-11695. SCM follower should not log NotLeaderException during Pipeline Report processing. (#7430)
5275ded92b HDDS-10133. Add a method to check key name in OMKeyRequest (#6012)
fd5c6d8cc2 HDDS-11689. Extract scheduled workflow for populate-cache (#7429)
889ba80766 HDDS-11653. Bump Ratis to 3.1.2 (#7427)
47ec4ddf87 HDDS-11671. Refer to website for supported versions (#7412)
10cac80f5c HDDS-11686. Use ozone image from GitHub in CI (#7425)
aa6da3e1ff HDDS-9781. Limited maxOpenFiles, disabled enableCompactionDag, and createCheckpointDirs when creating OMMetadataManager instance for bootstrapping (#7095)
9dd6a8301b HDDS-11645. Mark TestReconScmSnapshot#testExplicitRemovalOfNode as flaky
8e617dc10c HDDS-11672. Mark TestSnapshotBackgroundServices#testCompactionLogBackgroundService as flaky
d09e6d4da3 HDDS-11646. Mark TestXceiverClientMetrics#testMetrics as flaky
8e4a508155 HDDS-11668. Recon List Keys API: Reuse key prefix if parentID is the same (#7410)
ee63232860 HDDS-11684. Remove suppression of HiddenField (#7423)
a33d8a3e66 HDDS-10166. Replace GenericTestUtils temporary directories with `@TempDir` (#7399)
3a18a9d6e5 HDDS-11664. Hadoop download failure not reported as error (#7421)
47c2409d2f HDDS-64. OzoneClientException should extend IOException. (#7403)
27fcd0cb57 HDDS-11685. Use ozone-testkrb5 from GitHub (#7424)
5663971efe HDDS-11665. Minor optimizations on the write path (#7407)
cb81f0cc57 HDDS-11683. Skip shade in most integration checks (#7422)
952e0ec385 HDDS-11681. Bump Bouncy Castle to 1.79 (#7387)
2797c45a6c HDDS-11677. Bump sqlite-jdbc to 3.47.0.0 (#7413)
358534b83a HDDS-11675. Bump maven-site-plugin to 3.21.0 (#7414)
cf7924526f HDDS-11674. Bump junit to 5.11.3 (#7415)
6dd566f08c HDDS-11583. Use ozone-runner from GitHub in CI (#7409)
ef2bf98c37 HDDS-11669. In OmUtils.normalizeKey isDebugEnabled should be evaluated first (#7411)
a7e301472f HDDS-11660. Recon List Key API: Reduce object creation and buffering memory (#7405)
5d18b9cdec HDDS-11659. Improve HSync compatibility test (#7404)
4e603aa93c HDDS-11462. Enhancing DataNode I/O Monitoring Capabilities. (#7206)
18f6e8ab53 HDDS-11311. Added Compatibility test for HSync (#7400)
0415c0b4f7 HDDS-11649. Recon ListKeys API: Simplify filter predicates (#7395)
2547ac0438 HDDS-11652. Fix SCM start command in SCM-HA doc (#7398)
c0458398a8 HDDS-11623. Improve OM Ratis Configuration change log message (#7388)
2b1524b596 HDDS-11609. Switch to Recon v2 UI as the default UI (#7358)
efe589297c HDDS-11641. Allow testing Hadoop with custom docker images (#7393)
c055036b38 HDDS-11637. Compile failure is ignored in build check (#7389)
67e526111e HDDS-11563. Display OM/SCM service ID as Namespace in web UI (#7321)
0fb5e50a5e HDDS-11587. Ozone Manager not processing file put requests with multi-tenancy enabled (#7316)
786bb49f88 HDDS-11642. MutableQuantiles should be stopped (#7392)
76ec9b9d71 HDDS-11639. Upgrade ozone-runner to Rocky Linux 9.3 (#7391)
3bc3b8ad84 HDDS-11621. Fix missing HADOOP_ variables in MR acceptance test (#7375)
6f9db61895 HDDS-11200. Hsync client-side metrics (#7371)
58d144369c HDDS-10240. Cleanup zero-copy EC (#7340)
5b065d8ada HDDS-11638. Bump cyclonedx-maven-plugin to 2.9.0 (#7383)
c7a196f313 HDDS-11635. Memory leak when using Ozone FS via Hadoop FileContext API (#7382)
c9956a1641 HDDS-11601. Intermittent failure in acceptance balancer test. (#7343)
a737fc3122 HDDS-11619. Remove dependency on hadoop-shaded-guava (#7373)
c4d68573f7 HDDS-11584. Document ozone debug ldb command (#7313)
dded26e9dd HDDS-11588. Add main artifact jar to classpath file (#7324)
afed6d9143 HDDS-11558. Make OM client retry idempotent (#7329)
e85b32da2a HDDS-11591. Copy dependencies when building each module (#7325)
72e56d7508 HDDS-11601. Disable flaky EC balancer acceptance test
ab16cbe05d HDDS-11507. Add error information to log while handling ServiceException. (#7367)
980b960406 HDDS-11380. Make node decommission error message more comprehensive (#7155)
61c094f18e HDDS-11614. Speed up TestTransferLeadershipShell (#7370)
91188b3203 HDDS-11352. Remove Flaky annotation from TestOzoneManagerHAWithStoppedNodes using Ratis 3.1.1
faf133d24a HDDS-11220. Initialize block length using the chunk list from DataNode before seek (#7221)
91d41a0d1e HDDS-11465. Introducing Schema Versioning for Recon to Handle Fresh Installs and Upgrades. (#7213)
7a27db200a HDDS-11134. Create compatibility test for FSO bucket usage (#7350)
30906d1df4 HDDS-11612. Bump jnr-posix to 3.1.20 (#7360)
bed4aef528 HDDS-11611. Bump docker-maven-plugin to 0.45.1 (#7362)
e2c3d57089 HDDS-11610. Bump maven-dependency-plugin to 3.8.1 (#7361)
24c1000dd6 HDDS-11041. Add admin request filter for S3 requests and UGI support for GrpcOmTransport (#7268)
0b849981a9 HDDS-11594. Update batchPut buffer log for rocksdb. (#7356)
c013516658 HDDS-11608. Client should not retry invalid protobuf request (#7354)
32a8c093e3 HDDS-11160. Improve Insights page UI (#7327)
782ad62057 HDDS-11388. Fix unnecessary call to the DB for ContainerBalancer#getBalancerStatusInfo (#7224)
35b6a3aa94 HDDS-11600. Intermittent failure in repro due to ordering differences in builddef.lst (#7342)
e7bf154708 HDDS-11132. Revert client version bump done as part of HDDS-10983 (#7348)
ea5cbff436 HDDS-11602. Bump ozone-runner to 20241022-jdk17-1 (#7347)
3f98df5e35 HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property (#7328)
8568075ddb HDDS-11570. Fix HDDS Docs build failure with Hugo v0.135.0 (#7337)
721ae58a09 HDDS-11057. Enable reproducible builds (#6856)
86b7aaec9c HDDS-11205. Implement a search feature for users to locate keys pending Deletion within the OM Deleted Keys Insights section (#6969)
f7b428d784 HDDS-11503. Add Robot test to verify Container Balancer for EC containers. (#7311)
85eb89b94e HDDS-11483. Make s3g object get and put operation buffer configurable (#7233)
515977a4bf HDDS-11582. Bump body-parser to 1.20.3 (#7307)
9b66267de4 HDDS-11589. ReconSCMDBDefinition should be singleton. (#7323)
f784a84a04 HDDS-11578. Unify constants for RATIS_SNAPSHOT_DIR (#7310)
4670a5ef8c HDDS-11498. Improve SCM deletion efficiency. (#7249)
3fb2cf0000 HDDS-11108. Extract keywords for multipart upload tests (#7318)
4b24aa9950 HDDS-11545. [UI] Add OM and SCM ID information (#7287)
860e2699c4 HDDS-11538. Let coverage report link to java sources (#7280)
2139367b52 HDDS-11581. Remove duplicate ContainerStateMachine#RaftGroupId (#7312)
64e035d229 HDDS-11573. Remove lib/gson-2.10.1.jar (#7309)
ce07a3ccdf HDDS-11456. Require successful dependency/licence checks for acceptance/compile/kubernetes (#7209)
c579d06b33 HDDS-11574. Ozone client leak in TestS3SDKV1 (#7308)
c044b79bb1 HDDS-10390. MiniOzoneCluster to support S3 Gateway (#7281)
8eef589b2a HDDS-11557. Simplify DBColumnFamilyDefinition. (#7298)
4c77f6b9e7 HDDS-11562. Parameterize TestSCMNodeManager#testProcessLayoutVersion (#7300)
b51c4b326a HDDS-11572. Bump commons-io to 2.17.0 (#7305)
3a37870af9 HDDS-11571. Bump log4j2 to 2.24.1 (#7301)
494798c299 HDDS-11564. Mark TestBlockOutputStream#testWriteExactly... as flaky
fabf5122b0 HDDS-11569. Bump restrict-imports-enforcer-rule from 2.5.0 to 2.6.0 (#7303)
1e62a0a029 HDDS-11568. Bump commons-codec to 1.17.1 (#7304)
e9f92a77eb HDDS-11567. Bump common-custom-user-data-maven-extension to 2.0.1 (#7302)
cb44d5e6be HDDS-11555. SCMDBDefinition should be singleton. (#7296)
d47313487f HDDS-11486. Reduce log level for NativeLibraryNotLoadedException in SnapshotDiffManager (#7290)
3348d918c4 HDDS-11564. Mark TestBlockOutputStream as flaky
e2f2aeb2c6 HDDS-11548. Add some logging to the StateMachine (#7291)
523c8600da HDDS-11439. De-duplicate code for ReplicatedFileChecksumHelper and ECFileChecksumHelper (#7264)
05a409e695 HDDS-11519. Clean up unused lines in BlockOutputStream
ffe71985e6 HDDS-11544. Improve work with arrays (#7286)
5657604486 HDDS-11556. Add a getTypeClass method to Codec. (#7295)
256aad9df2 HDDS-11546. Add regex operation to filter option of ldb scan command. (#7289)
7ef7de200b HDDS-11482. EC Checksum throws IllegalArgumentException because the buffer limit is negative (#7230)
77c17dfdcf HDDS-11551. Provide details about integration check failure (#7294)
911a583e72 HDDS-8188. Support max allowed length in response of ozone admin container list (#7181)
7f2e0e37de HDDS-11554. OMDBDefinition should be singleton. (#7292)
170761cd48 HDDS-11547. Make MAVEN_OPTS optional (#7288)
4846e97b1a HDDS-11543. Track OzoneClient object leaks via LeakDetector framework. (#7285)
e00f7ae960 HDDS-11159. Improve Containers page UI (#7267)
cfda951c91 HDDS-11520. Fix Delete pending directories key mapping (#7269)
2e3de8ab17 HDDS-11476. Implement lesser/greater operation for --filter option of ldb scan command (#7222)
06ccdb3f8a HDDS-11526. Fix hdds.datanode.metadata.rocksdb.cache.size default value mismatch (#7284)
b3afaec5c1 HDDS-11535. Incomplete SCM roles table header (#7278)
ed2a073499 HDDS-11536. Bump macOS runner version to macos-13 (#7279)
1887f8339b HDDS-11537. Bump frontend-maven-plugin to 1.15.1 (#7276)
1f1e61833c HDDS-6776. Cleanup TestSCMSafeModeManager (#7272)
4bee3e963a HDDS-11534. Bump cyclonedx-maven-plugin to 2.8.2 (#7277)
789fb53f9e HDDS-11533. Bump maven-gpg-plugin to 3.2.7 (#7275)
eb26677e61 HDDS-11268. Add --table mode for OM/SCM Roles CLI (#7016)
28ea4805a9 HDDS-11527. Avoid unnecessary duplicate build (#7270)
30da31f1fa HDDS-3498. Shutdown datanode if address is already in use (#7256)
2401d27f68 HDDS-11046. Coverage decreased due to running tests with Java 17 (#7263)
78d8418303 HDDS-11524. Bump snappy-java to 1.1.10.7 (#7202)
8747c0ea7a HDDS-11518. Recon OmDB Insights show isKey=true for directories (#7260)
5d2bbc3dd7 HDDS-11480. Refactor OM volume response tests (#7265)
31f9f2c952 HDDS-11517. Update version to 2.0.0-SNAPSHOT (#7258)
a0f0872fc8 HDDS-11444. Make Datanode Command metrics consistent across all commands (#7191)
d3b63c681a HDDS-11492. Directory deletion get stuck having millions of directory (#7254)
f52f0afd73 HDDS-11127. [hsync] Improve test coverage for XceiverClientRatis.java (#7225)
360fea5b2e HDDS-11494. Improve the duration option of freon ombg (#7246)
10d3b213e8 HDDS-11504. Update Ratis to 3.1.1. (#7257)
ce46297e85 HDDS-11162. Improve Disk Usage page UI (#7214)
c91f1c7f8d HDDS-11491. Avoid sharing clientId among deleting services (#7250)
b0943d5f86 HDDS-11501. Improve logging in XceiverServerRatis (#7252)
55925ab623 HDDS-11502. Class path contains multiple SLF4J providers (#7255)
d0ad8362f6 HDDS-11472. Avoid recreating external access authorizer on OM state reload (#7238)
254db9e723 HDDS-11500. RootCARotationManager cancelling wrong task in notifyStatusChanged (#7251)
1e6e4b34ff HDDS-11499. Remove redundant code from ECReconstructionCoordinator. (#7248)
adb2821a52 HDDS-11490. Bump rollup to 3.29.5 (#7232)
189a9fe420 HDDS-11484. Validate javadoc in CI (#7245)
64a29c62c3 HDDS-11497. Bump commons-configuration2 to 2.11.0 (#7242)
95cfadd15c HDDS-11496. Bump maven-install-plugin to 3.1.3 (#7244)
0a999cf77d HDDS-11493. Bump sqlite-jdbc to 3.46.1.3 (#7243)
a214a3153b HDDS-11329. Update Ozone images to Rocky Linux-based runner (#7241)
56ddb85d94 HDDS-11371. Handle cases where OM does not have getServerDefaults() implemented. (#7130)
b5097c76da HDDS-11347. Add rocks_tools_native lib check in Ozone CLI checknative subcommand (#7101)
fb0bf77265 HDDS-11489. Bump maven-site-plugin to 3.20.0 (#7226)
70e6e40a3c HDDS-11122. Fix javadoc warnings (#7234)
acf3fdc7c2 HDDS-11458. Selective checks: trigger checkstyle for properties file changes (#7196)
6b872073b9 HDDS-11469. Statistics of Pipeline and Container (#7217)
1b8468ba8e HDDS-11411. Snapshot garbage collection should not run when the keys are moved from a deleted snapshot to the next snapshot in the chain (#7193)
1f86ce80bd HDDS-10617. Unexpected number of files in ITestS3AContractGetFileStatusV1List (#7208)
73a3bccba3 HDDS-11467. Bump vite to 4.5.5 (#7212)
d45aa1d46b HDDS-11460. Bump express to 4.21.0 (#7197)
e2e30b856d HDDS-11354. Intermittent failure in TestOzoneManagerSnapshotAcl#testLookupKeyWithNotAllowedUserForPrefixAcl (#7205)
0fcb64564d HDDS-11477. [doc] Add configuration description for datanode docs (#7223)
3598ee3d42 HDDS-11464. Removed unused constants from OzoneConsts. (#7207)
8c0b54ef0c HDDS-11408. Snapshot rename table entries are propagated incorrectly on snapshot deletes (#7200)
719bdf9c94 HDDS-11396. NPE due to empty Handler#clusterId (#7145)
40c4001ed0 HDDS-10479. Add ozone admin ratis local raftMetaConf (#7170)
45f9138734 HDDS-11394. Fix pipeline close --all command (#7138)
2b196d1979 HDDS-11468. Enabled DB sync button (#7216)
d3899d2cc3 Clean up files created after TestKeyValueHandlerWithUnhealthyContainer#testMarkContainerUnhealthyInFailedVolume (#7219)
70b8dd5ea3 HDDS-11157. Improve Datanodes page UI (#7168)
151709a5f7 HDDS-11446. Downgrade picocli to 4.7.5 due to regression (#7215)
7a26afff5a HDDS-11158. Improve Pipelines page UI (#7171)
c365aa025f HDDS-11181. Cleanup of unnecessary try-catch blocks (#7210)
88dd4369e2 HDDS-11423. Implement equals operation for --filter option to ozone ldb scan (#7167)
e0060a8f50 HDDS-11196. Improve SCM WebUI Display (#6960)
22ddfb951a Revert "HDDS-11456. Require successful dependency/licence checks for acceptance/compile/kubernetes (#7192)"
9f5bf43910 HDDS-11457. Internal error on S3 CompleteMultipartUpload if parts are not specified (#7195)
10c47a16e4 HDDS-11459. Bump develocity-maven-extension to 1.22.1 (#7201)
50f256300e HDDS-11419. Fix waitForCheckpointDirectoryExist log message (#7199)
a7d7e37fe1 HDDS-11456. Require successful dependency/licence checks for acceptance/compile/kubernetes (#7192)
5feb9eacd8 HDDS-11453. OmSnapshotPurge should be in a different ozone manager double buffer batch (#7188)
703c4d50f8 HDDS-10984. Tool to restore SCM certificates from RocksDB. (#6781)
d221065705 HDDS-11440. Add a lastTransactionInfo field in SnapshotInfo to check for transactions in flight on the snapshot (#7179)
e57370124a HDDS-11448. Improve documentation in ContainerStateMachine (#7183)
0e49f7adcd HDDS-11449. Remove unnecessary log from client console. (#7184)
cd251f23a9 HDDS-11438. Ensure DataInputBuffer is closed in OMPBHelper#convert (#7182)
4b47812a2d HDDS-11389. Incorrect number of deleted containers shown in Recon UI. (#7149)
0915f0b1b8 HDDS-10985. EC Reconstruction failed because the size of currentChunks was not equal to checksumBlockDataChunks. (#7009)
0f16195730 HDDS-11416. refactor ratis submit request avoid code duplicate (#7166)
86fe920030 HDDS-11376. Improve ReplicationSupervisor to record replication metrics (#7140)
883a63f472 HDDS-11441. ozone sh key put should only accept positive expectedGeneration (#7180)
33dbd4a385 HDDS-11357. Datanode Usageinfo Support Display Pipeline. (#7105)
9477aa63fd HDDS-11436. Minor update in Recon API handling. (#7178)
8ca33c718b HDDS-11414. Key listing for FSO buckets fails with forward client (#7161)
f1ebd39c98 HDDS-11435. Bump sqlite-jdbc to 3.46.1.0 (#7174)
aaf8bd00ab HDDS-11434. Bump log4j2 to 2.24.0 (#7176)
3510ce7cca HDDS-11433. Bump Jetty to 9.4.56.v20240826 (#7175)
0047cd2ba3 HDDS-11400. Bump maven-core to 3.9.9 (#7144)
274da83cfe HDDS-10488. Datanode OOM due to run out of mmap handler (#6690)
7a452ca2e8 HDDS-11391. Addendum to fix test failure.
7e1d9b03a5 HDDS-11145. ozone admin om cancelprepare --service-id improvement (#7159)
6888cf2150 HDDS-11383. Improve read key dashboard to include add the read key related OM metrics. (#7131)
3e0d76cba2 HDDS-11369. [hsync] Remove KeyOutputStreamSemaphore logs (#7136)
b23981cbda HDDS-11342. [hsync] Add a config as HBase-related features master switch (#7126)
3e1188aa28 HDDS-11285. cli to trigger quota repair and status (#7104)
2e339787d1 HDDS-11401. Code cleanup in DatanodeStateMachine (#7146)
f563d676dc HDDS-11391. Frequent Ozone DN Crashes During OM + DN Decommission with Freon (#7154)
18b28d20e5 HDDS-11312. [hsync] Added upgrade tests (#7110)
b29beb3364 HDDS-11350. NullPointerException thrown on checking container balancer status (#7134)
111b9df4e6 HDDS-11407. Use OMLayoutFeature.HBASE_SUPPORT for HSYNC (#7152)
966b8d0c5a HDDS-11390. Removed hsync and hflush capability check in ContentGenerator (#7153)
877504aee1 HDDS-11156. Improve Buckets page UI (#7100)
814f78f9ff HDDS-11392. ChecksumByteBufferImpl's static initializer fails with java 17+ (#7135)
b5e1a8b171 HDDS-11398. Bump commons-compress to 1.27.1 (#7142)
a8e3ea97c7 HDDS-11397. Bump Jersey2 to 2.45 (#7141)
5992837af4 HDDS-11399. Bump maven-deploy-plugin to 3.1.3 (#7143)
47564bb47d HDDS-11359. Intermittent timeout in TestPipelineManagerMXBean#testPipelineInfo (#7132)
cc4e026d59 HDDS-11304. Make up for the missing functionality in CommandDispatcher (#7062)
2d372f6abd HDDS-11339. Let PrometheusServlet rely on periodically published metrics (#7092)
f22c6f8dfc HDDS-11164. Improve Navbar UI (#7088)
23211c1932 HDDS-11381. Adding logging for sortByDistanceCost in NetworkTopologyImpl (#7133)
3e9cdb609b HDDS-11378. Allow disabling OM version-specific feature via config (#7129)
23f3e5b2e1 HDDS-11152. OMDoubleBuffer error when handling snapshot's background operations (#7112)
5659b7e90e HDDS-11375. DN startup fails due to illegal configuration of raft.grpc.message.size.max (#7128)
41d81479e4 HDDS-11368. Remove dependency on Babel in Vite (#7119)
0bd8ba1e6c HDDS-11372. No coverage for org.apache.ozone packages (#7124)
3bd237d1bd HDDS-11325. (addendum) Intermittent failure in TestBlockOutputStreamWithFailures#testContainerClose (#7121)
830629009d HDDS-11373. Log for EC reconstruction command lists the missing indexes as ASCII control characters (#7123)
dab15385dc HDDS-11216. Replace HAUtils#buildCAX509List usages with other direct usages (#6981)
51a5fb9422 Revert "HDDS-11235. Spare InfoBucket RPC call in FileSystem#mkdir() call. (#6990)" (#7122)
fab56b44a0 HDDS-11229. Chain optionals in Recon Insight (#7064)
2236041f3a HDDS-11365. Fix the NOTICE file (#7120)
2e30dc182c HDDS-11190. Add --fields option to ldb scan command (#6976)
0b75cb00be HDDS-11251. Deprecate definitions and remove listTrash and recoverTrash APIs (#7060)
be34303650 HDDS-9198. Maintain local cache in OMSnapshotPurgeRequest to get updated snapshotInfo and pass the same to OMSnapshotPurgeResponse (#7045)
c07b408bdf HDDS-11208. Change RatisBlockOutputStream to use HDDS-11174. (#7072)
8f8d8094c7 HDDS-11309. Increase CONTAINER_STATE Column Length in UNHEALTHY_CONTAINERS to Avoid Truncation (#7071)
350a340277 HDDS-11364. Bump jgraphx to 3.9.12 (#7116)
45b7056c68 HDDS-11363. Bump develocity-maven-extension to 1.22 (#7115)
9dd18f1611 HDDS-11362. Bump snappy-java to 1.1.10.6 (#7114)
637cb91346 HDDS-11361. Bump Jersey2 to 2.44 (#7113)
Conflicts:
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
Modified during conflict:
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ReconcileContainerTask.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
.asf.yaml | 2 +-
.github/workflows/ci.yml | 168 +-
.github/workflows/intermittent-test-check.yml | 4 +-
.github/workflows/populate-cache.yml | 5 +-
.github/workflows/scheduled-cache-update.yml | 27 +
.mvn/extensions.xml | 4 +-
NOTICE.txt | 2 +-
SECURITY.md | 14 +-
dev-support/ci/maven-settings.xml | 35 +
dev-support/ci/selective_ci_checks.bats | 42 +-
dev-support/ci/selective_ci_checks.sh | 7 +-
dev-support/pom.xml | 96 +
hadoop-hdds/annotations/pom.xml | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 53 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 75 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 3 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../hdds/scm/storage/AbstractCommitWatcher.java | 1 -
.../hdds/scm/storage/BlockDataStreamOutput.java | 1 -
.../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 160 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 34 +-
.../hdds/scm/storage/ExtendedInputStream.java | 1 -
.../hdds/scm/storage/MultipartInputStream.java | 27 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 4 +-
.../ozone/client/io/BlockInputStreamFactory.java | 1 -
.../client/io/BlockInputStreamFactoryImpl.java | 1 -
.../hadoop/ozone/client/io/ECBlockInputStream.java | 5 -
.../ozone/client/io/ECBlockInputStreamFactory.java | 1 -
.../client/io/ECBlockInputStreamFactoryImpl.java | 1 -
.../io/ECBlockReconstructedStripeInputStream.java | 2 +-
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 40 +
hadoop-hdds/common/pom.xml | 4 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 50 +-
.../java/org/apache/hadoop/hdds/JavaUtils.java | 14 +-
.../hadoop/hdds/annotation/InterfaceStability.java | 6 +-
.../hadoop/hdds/cli/ExtensibleParentCommand.java | 56 +
.../org/apache/hadoop/hdds/cli/GenericCli.java | 45 +-
.../hadoop/hdds/cli/SubcommandWithParent.java | 30 -
.../hadoop/hdds/client/DecommissionUtils.java | 2 -
.../hadoop/hdds/client/ECReplicationConfig.java | 5 +
.../hadoop/hdds/client/RatisReplicationConfig.java | 5 +
.../hadoop/hdds/client/ReplicationConfig.java | 2 +
.../hdds/client/StandaloneReplicationConfig.java | 5 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 176 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 6 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +
.../java/org/apache/hadoop/hdds/scm/ScmInfo.java | 23 +-
.../hadoop/hdds/scm/client/ClientTrustManager.java | 0
.../apache/hadoop/hdds/scm/client/ScmClient.java | 23 +-
.../hadoop/hdds/scm/container/ContainerID.java | 2 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 3 +-
.../hdds/scm/container/ContainerListResult.java | 57 +
.../scm/container/ReplicationManagerReport.java | 2 -
.../common/helpers/ContainerWithPipeline.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 1 -
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 5 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 6 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +-
.../hadoop/hdds/scm/pipeline/PipelineID.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 24 +-
.../x509/certificate/client/CertificateClient.java | 40 +-
.../certificate/utils/CertificateSignRequest.java | 11 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 2 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 -
.../org/apache/hadoop/hdds/utils/LeakDetector.java | 13 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 13 -
.../apache/hadoop/hdds/utils/db/BooleanCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/Codec.java | 3 +
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 6 +-
.../hadoop/hdds/utils/db/DelegatedCodec.java | 22 +-
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 5 +
.../apache/hadoop/hdds/utils/db/Proto2Codec.java | 7 +
.../apache/hadoop/hdds/utils/db/Proto3Codec.java | 7 +
.../apache/hadoop/hdds/utils/db/ShortCodec.java | 5 +
.../hadoop/hdds/utils/db/StringCodecBase.java | 5 +
.../org/apache/hadoop/hdds/utils/db/UuidCodec.java | 5 +
.../org/apache/hadoop/ozone/ClientVersion.java | 4 -
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 69 +-
.../apache/hadoop/ozone/OzoneManagerVersion.java | 5 +
.../org/apache/hadoop/ozone/common/Checksum.java | 91 +-
.../ozone/common/ChecksumByteBufferImpl.java | 14 +-
.../apache/hadoop/ozone/common/ChecksumCache.java | 122 +
.../apache/hadoop/ozone/common/ChunkBuffer.java | 8 +-
.../common/ChunkBufferImplWithByteBuffer.java | 7 +-
.../common/ChunkBufferImplWithByteBufferList.java | 6 +-
.../ozone/common/IncrementalChunkBuffer.java | 3 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 50 +
.../ozone/container/common/helpers/BlockData.java | 16 +-
.../container/common/helpers/ChunkInfoList.java | 5 +-
.../hadoop/ozone/grpc/metrics/GrpcMetrics.java | 3 +
.../upgrade/LayoutVersionInstanceFactory.java | 14 +-
.../hadoop/ozone/upgrade/LayoutVersionManager.java | 1 -
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 12 +-
.../hadoop/ozone/upgrade/VersionFactoryKey.java | 2 +-
.../org/apache/hadoop/ozone/util/MetricUtil.java | 16 +
.../hadoop/ozone/util/PerformanceMetrics.java | 15 +-
.../ozone/util/PerformanceMetricsInitializer.java | 9 +-
.../main/resources/hdds-version-info.properties | 3 -
.../common/src/main/resources/ozone-default.xml | 90 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 23 +-
.../container/common/helpers/TestExcludeList.java | 16 +-
.../hadoop/hdds/utils/MockGatheringChannel.java | 67 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 38 +-
.../hadoop/ozone/common/TestChecksumCache.java | 75 +
.../ozone/container/ContainerTestHelper.java | 22 +-
hadoop-hdds/config/pom.xml | 4 +-
.../hadoop/hdds/conf/ConfigurationSource.java | 2 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../java/org/apache/hadoop/ozone/DNMXBean.java | 28 +
.../java/org/apache/hadoop/ozone/DNMXBeanImpl.java | 49 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 29 +-
.../apache/hadoop/ozone/HddsPolicyProvider.java | 4 +-
.../checksum/DNContainerOperationClient.java | 8 +-
.../container/checksum/ReconcileContainerTask.java | 10 +
.../helpers/BlockDeletingServiceMetrics.java | 32 +-
.../common/helpers/CommandHandlerMetrics.java | 3 +
.../container/common/helpers/ContainerMetrics.java | 9 +-
.../common/impl/BlockDeletingService.java | 9 +-
.../ozone/container/common/impl/ContainerSet.java | 146 +-
.../container/common/impl/HddsDispatcher.java | 8 +-
.../common/impl/OpenContainerBlockMap.java | 2 +-
.../ContainerDeletionChoosingPolicyTemplate.java | 2 +-
.../common/interfaces/ContainerDispatcher.java | 1 -
.../ozone/container/common/interfaces/Handler.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 1 -
.../common/statemachine/EndpointStateMachine.java | 15 +-
.../CloseContainerCommandHandler.java | 31 +-
.../ClosePipelineCommandHandler.java | 16 +-
.../commandhandler/CommandDispatcher.java | 17 +-
.../CreatePipelineCommandHandler.java | 16 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 34 +-
.../DeleteContainerCommandHandler.java | 33 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 16 +-
.../ReconstructECContainersCommandHandler.java | 26 +-
.../RefreshVolumeUsageCommandHandler.java | 16 +-
.../ReplicateContainerCommandHandler.java | 31 +-
.../SetNodeOperationalStateCommandHandler.java | 26 +-
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../states/endpoint/VersionEndpointTask.java | 3 +-
.../transport/server/GrpcXceiverService.java | 19 +-
.../common/transport/server/XceiverServerGrpc.java | 26 +-
.../server/ratis/ContainerStateMachine.java | 142 +-
.../transport/server/ratis/XceiverServerRatis.java | 88 +-
.../ozone/container/common/volume/HddsVolume.java | 22 +-
.../container/common/volume/MutableVolumeSet.java | 5 +-
.../container/common/volume/VolumeIOStats.java | 57 +-
.../ozone/container/common/volume/VolumeInfo.java | 17 +-
.../container/common/volume/VolumeInfoMetrics.java | 8 +
.../ozone/container/common/volume/VolumeUsage.java | 12 +-
.../reconstruction/ECContainerOperationClient.java | 32 +-
.../ECReconstructionCoordinator.java | 15 +-
.../ECReconstructionCoordinatorTask.java | 10 +
.../container/keyvalue/KeyValueContainer.java | 1 -
.../container/keyvalue/KeyValueContainerData.java | 2 -
.../ozone/container/keyvalue/KeyValueHandler.java | 92 +-
.../container/keyvalue/helpers/BlockUtils.java | 5 +-
.../container/keyvalue/helpers/ChunkUtils.java | 70 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 5 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 9 +
.../keyvalue/impl/ChunkManagerFactory.java | 1 -
.../keyvalue/impl/FilePerBlockStrategy.java | 14 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 12 +-
.../keyvalue/impl/MappedBufferManager.java | 122 +
.../keyvalue/impl/StreamDataChannelBase.java | 4 +-
.../keyvalue/interfaces/BlockManager.java | 3 +
.../container/metadata/AbstractDatanodeStore.java | 210 +-
.../ozone/container/metadata/AbstractRDBStore.java | 135 +
.../ozone/container/metadata/DBStoreManager.java | 71 +
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 -
.../metadata/DatanodeSchemaThreeDBDefinition.java | 10 -
.../metadata/DatanodeSchemaTwoDBDefinition.java | 11 -
.../ozone/container/metadata/DatanodeStore.java | 49 +-
.../metadata/SchemaOneChunkInfoListCodec.java | 8 +-
.../container/metadata/SchemaOneKeyCodec.java | 5 +
.../metadata/WitnessedContainerDBDefinition.java | 71 +
.../metadata/WitnessedContainerMetadataStore.java | 34 +
.../WitnessedContainerMetadataStoreImpl.java | 78 +
.../ozoneimpl/BackgroundContainerDataScanner.java | 1 +
.../container/ozoneimpl/ContainerController.java | 11 +-
.../ozoneimpl/ContainerDataScannerMetrics.java | 11 +
.../ozone/container/ozoneimpl/ContainerReader.java | 3 +-
.../ozoneimpl/OnDemandContainerDataScanner.java | 3 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 53 +-
.../replication/AbstractReplicationTask.java | 4 +
.../container/replication/ContainerImporter.java | 2 +-
.../replication/GrpcReplicationService.java | 35 +-
.../container/replication/ReplicationServer.java | 29 +-
.../replication/ReplicationSupervisor.java | 122 +-
.../replication/ReplicationSupervisorMetrics.java | 38 +-
.../container/replication/ReplicationTask.java | 10 +
.../replication/SendContainerRequestHandler.java | 6 +-
.../ozone/container/stream/StreamingSource.java | 4 +-
.../commands/ReconstructECContainersCommand.java | 4 +-
...inerDatanodeProtocolClientSideTranslatorPB.java | 1 -
.../webapps/hddsDatanode/dn-overview.html | 28 +-
.../resources/webapps/hddsDatanode/dn-scanner.html | 47 +
.../src/main/resources/webapps/hddsDatanode/dn.js | 100 +-
.../main/resources/webapps/hddsDatanode/index.html | 9 +-
.../resources/webapps/hddsDatanode/iostatus.html | 76 +
.../ozone/container/common/ContainerTestUtils.java | 8 +-
.../container/common/TestDatanodeStateMachine.java | 15 +-
.../common/helpers/TestContainerUtils.java | 3 +-
.../common/helpers/TestDatanodeIdYaml.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 9 +-
.../TestCloseContainerCommandHandler.java | 36 +-
.../TestDeleteContainerCommandHandler.java | 59 +-
.../TestReconstructECContainersCommandHandler.java | 139 +
.../TestReplicateContainerCommandHandler.java | 118 +
.../common/volume/TestReservedVolumeSpace.java | 12 +
.../TestVolumeIOStatsWithPrometheusSink.java | 9 +-
.../common/volume/TestVolumeSetDiskChecks.java | 1 +
.../container/keyvalue/TestKeyValueContainer.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 9 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 6 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 4 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 56 +
.../keyvalue/impl/CommonChunkManagerTestCases.java | 25 +
.../keyvalue/impl/TestMappedBufferManager.java | 46 +
.../container/ozoneimpl/TestOzoneContainer.java | 31 +-
.../replication/TestGrpcReplicationService.java | 14 +-
.../TestGrpcReplicationServiceWithZeroCopy.java | 31 -
.../replication/TestReplicationSupervisor.java | 231 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 9 +-
.../TestReconstructionECContainersCommands.java | 13 +-
hadoop-hdds/crypto-api/pom.xml | 4 +-
hadoop-hdds/crypto-default/pom.xml | 4 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
hadoop-hdds/docs/content/concept/Datanodes.md | 12 +
hadoop-hdds/docs/content/concept/Datanodes.zh.md | 12 +
hadoop-hdds/docs/content/feature/Quota.md | 2 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 4 +-
.../docs/content/feature/dn-merge-rocksdb.md | 8 +-
.../docs/content/feature/dn-merge-rocksdb.zh.md | 6 +-
hadoop-hdds/docs/content/interface/CSI.md | 2 +-
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
hadoop-hdds/docs/content/security/GDPR.md | 2 +-
hadoop-hdds/docs/content/security/GDPR.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 2 +-
.../docs/content/security/SecureOzone.zh.md | 2 +-
.../docs/content/security/SecuringDatanodes.md | 2 +-
.../docs/content/security/SecuringDatanodes.zh.md | 2 +-
.../docs/content/security/SecuringOzoneHTTP.md | 2 +-
.../docs/content/security/SecuringOzoneHTTP.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringS3.md | 2 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 2 +-
.../docs/content/security/SecuringTDE.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecurityAcls.md | 2 +-
.../docs/content/security/SecurityAcls.zh.md | 2 +-
.../docs/content/security/SecurityWithRanger.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
hadoop-hdds/docs/content/tools/Debug.md | 473 +++
hadoop-hdds/docs/content/tools/Debug.zh.md | 466 +++
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../apache/ozone/erasurecode/CodecRegistry.java | 7 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 17 -
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 1 -
.../SCMSecurityProtocolClientSideTranslatorPB.java | 1 -
.../container/common/helpers/MoveDataNodePair.java | 1 +
...lockLocationProtocolClientSideTranslatorPB.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 42 +-
.../hadoop/hdds/security/OzoneSecretKey.java | 53 +-
.../hadoop/hdds/security/OzoneSecretManager.java | 5 +-
.../hadoop/hdds/security/token/TokenVerifier.java | 3 +-
.../hdds/security/x509/certificate/CertInfo.java | 4 +-
.../certificate/authority/DefaultApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 2 -
.../client/DefaultCertificateClient.java | 57 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 89 +
.../hadoop/hdds/server/ServiceRuntimeInfo.java | 16 +-
.../hadoop/hdds/server/ServiceRuntimeInfoImpl.java | 6 -
.../hadoop/hdds/server/http/HttpServer2.java | 4 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 8 +-
.../hadoop/hdds/server/http/PrometheusServlet.java | 3 -
.../hadoop/hdds/utils/DBCheckpointServlet.java | 14 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 116 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 8 +-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 12 +-
.../hadoop/hdds/utils/db/ByteArrayCodec.java | 5 +
.../hadoop/hdds/utils/db/ByteStringCodec.java | 5 +
.../hdds/utils/db/DBColumnFamilyDefinition.java | 21 +-
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 22 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +-
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 133 +
.../hadoop/hdds/utils/db/RDBCheckpointUtils.java | 2 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 4 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 30 +-
.../apache/hadoop/ozone/audit/package-info.java | 2 +-
.../src/main/resources/webapps/static/ozone.css | 23 +
.../src/main/resources/webapps/static/ozone.js | 14 +-
.../resources/webapps/static/templates/jvm.html | 13 +-
.../resources/webapps/static/templates/menu.html | 2 +
.../webapps/static/templates/overview.html | 6 +-
.../client/CertificateClientTestImpl.java | 22 +-
.../apache/hadoop/hdds/server/TestOzoneAdmins.java | 125 +
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 6 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 21 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 16 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 13 +-
.../src/main/proto/ScmAdminProtocol.proto | 22 +-
hadoop-hdds/interface-client/pom.xml | 14 +-
.../interface-client/src/main/proto/hdds.proto | 4 +-
.../interface-client/src/main/resources/proto.lock | 12 +
hadoop-hdds/interface-server/pom.xml | 12 +-
.../interface-server/src/main/resources/proto.lock | 24 +
hadoop-hdds/managed-rocksdb/pom.xml | 4 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 13 +
.../utils/db/managed/ManagedRocksObjectUtils.java | 7 +-
hadoop-hdds/pom.xml | 75 +-
hadoop-hdds/rocks-native/pom.xml | 4 +-
.../hadoop/hdds/utils/NativeLibraryLoader.java | 42 +-
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 50 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 4 +-
.../ozone/compaction/log/CompactionFileInfo.java | 12 +
.../ozone/compaction/log/CompactionLogEntry.java | 3 +-
.../org/apache/ozone/rocksdiff/CompactionNode.java | 7 +
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 203 +-
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 96 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 299 +-
.../apache/ozone/rocksdiff/TestRocksDiffUtils.java | 121 +
hadoop-hdds/server-scm/pom.xml | 5 +-
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 6 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 14 +-
.../block/DatanodeDeletedBlockTransactions.java | 3 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 108 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 7 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 8 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 177 +-
.../hdds/scm/container/ContainerManager.java | 9 +-
.../hdds/scm/container/ContainerManagerImpl.java | 6 +
.../hdds/scm/container/ContainerReportHandler.java | 7 +
.../balancer/AbstractFindTargetGreedy.java | 9 +-
.../scm/container/balancer/ContainerBalancer.java | 20 +-
.../balancer/ContainerBalancerMetrics.java | 52 +-
.../ContainerBalancerSelectionCriteria.java | 16 +-
.../balancer/ContainerBalancerStatusInfo.java | 19 +
.../container/balancer/ContainerBalancerTask.java | 301 +-
.../ContainerBalancerTaskIterationStatusInfo.java | 166 +-
.../scm/container/balancer/ContainerMoveInfo.java | 60 +
.../hdds/scm/container/balancer/DataMoveInfo.java | 60 +
.../scm/container/balancer/FindSourceGreedy.java | 9 +-
.../scm/container/balancer/FindSourceStrategy.java | 9 +
.../scm/container/balancer/FindTargetStrategy.java | 9 +
.../hdds/scm/container/balancer/IterationInfo.java | 47 +
.../algorithms/SCMContainerPlacementRackAware.java | 3 +-
.../container/placement/metrics/SCMNodeMetric.java | 3 +-
.../replication/ContainerHealthResult.java | 1 -
.../replication/ContainerReplicaCount.java | 2 +-
.../replication/ContainerReplicaPendingOps.java | 8 +-
.../LegacyRatisContainerReplicaCount.java | 142 -
.../replication/LegacyReplicationManager.java | 2581 ---------------
.../replication/RatisContainerReplicaCount.java | 12 +-
.../container/replication/ReplicationManager.java | 96 +-
.../replication/ReplicationManagerMetrics.java | 50 +-
.../container/replication/health/HealthCheck.java | 1 -
.../scm/container/states/ContainerAttribute.java | 8 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 7 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 3 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcService.java | 12 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 1 -
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 28 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 8 +
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 11 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 38 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 2 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 5 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 18 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 2 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 11 +
.../hdds/scm/node/NodeDecommissionManager.java | 20 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 2 -
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 2 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 47 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 1 -
...lockLocationProtocolServerSideTranslatorPB.java | 3 +-
...inerLocationProtocolServerSideTranslatorPB.java | 14 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 287 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 51 +-
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 16 +
.../hdds/scm/safemode/SafeModeRuleFactory.java | 131 +
.../hdds/scm/security/RootCARotationManager.java | 8 +-
.../hdds/scm/security/SecretKeyManagerService.java | 3 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 228 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 16 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 3 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 7 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 10 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 -
.../hdds/scm/server/StorageContainerManager.java | 84 +-
.../server/upgrade/FinalizationManagerImpl.java | 17 +-
.../upgrade/SCMUpgradeFinalizationContext.java | 9 +-
.../main/resources/webapps/scm/scm-overview.html | 149 +-
.../src/main/resources/webapps/scm/scm.js | 85 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 26 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 42 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 71 +-
.../scm/container/TestContainerManagerImpl.java | 3 +-
.../scm/container/TestContainerReportHandler.java | 3 +-
.../scm/container/TestContainerStateManager.java | 3 +-
.../TestIncrementalContainerReportHandler.java | 5 +-
.../scm/container/TestUnknownContainerReport.java | 3 +-
.../balancer/ContainerBalancerConfigBuilder.java | 46 +
.../hdds/scm/container/balancer/MockedSCM.java | 39 +-
.../container/balancer/TestContainerBalancer.java | 17 +
.../TestContainerBalancerDatanodeNodeLimit.java | 306 +-
.../balancer/TestContainerBalancerStatusInfo.java | 219 +-
.../balancer/TestContainerBalancerTask.java | 285 +-
.../replication/TestLegacyReplicationManager.java | 3489 --------------------
.../replication/TestReplicationManager.java | 3 -
.../replication/TestReplicationManagerMetrics.java | 6 -
.../TestReplicationManagerScenarios.java | 3 -
.../hdds/scm/ha/TestReplicationAnnotation.java | 7 +
.../hadoop/hdds/scm/ha/TestSCMRatisServerImpl.java | 108 +
.../ha/TestStatefulServiceStateManagerImpl.java | 2 +-
.../scm/metadata/OldPipelineIDCodecForTesting.java | 4 +
.../OldX509CertificateCodecForTesting.java | 5 +
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 147 -
.../hdds/scm/node/TestNodeDecommissionManager.java | 66 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 16 +-
.../TestPipelineDatanodesIntersection.java | 3 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 7 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 9 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 3 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 3 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 3 +-
.../pipeline/TestWritableECContainerProvider.java | 3 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 15 +-
.../TestOneReplicaPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 262 +-
.../hdds/scm/safemode/TestSafeModeRuleFactory.java | 85 +
.../scm/server/TestSCMClientProtocolServer.java | 54 +
hadoop-hdds/test-utils/pom.xml | 4 +-
.../org/apache/ozone/test/GenericTestUtils.java | 42 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 4 +
.../java/org/apache/ozone/test/MetricsAsserts.java | 7 +-
hadoop-hdds/tools/pom.xml | 20 +-
.../apache/hadoop/hdds/cli/AdminSubcommand.java | 23 +
.../apache/hadoop/hdds/cli/DebugSubcommand.java | 23 +
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 44 +-
.../apache/hadoop/hdds/cli/RepairSubcommand.java | 23 +
.../hdds/scm/cli/ContainerBalancerCommands.java | 13 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 71 +-
.../hdds/scm/cli/ContainerOperationClient.java | 28 +-
.../hdds/scm/cli/ReplicationManagerCommands.java | 13 +-
.../hadoop/hdds/scm/cli/SafeModeCommands.java | 12 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 12 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 12 +-
.../hdds/scm/cli/container/ContainerCommands.java | 15 +-
.../hdds/scm/cli/container/ListSubcommand.java | 61 +-
.../hdds/scm/cli/datanode/DatanodeCommands.java | 13 +-
.../hdds/scm/cli/datanode/StatusSubCommand.java | 10 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 9 +
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 18 +-
.../hdds/scm/cli/pipeline/PipelineCommands.java | 12 +-
.../org/apache/hadoop/hdds/util/DurationUtil.java | 52 +
.../org/apache/hadoop/hdds/util/package-info.java | 22 +
.../datanode/TestContainerBalancerSubCommand.java | 419 ++-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 3 +
.../cli/pipeline/TestClosePipelinesSubCommand.java | 178 +
.../apache/hadoop/hdds/util/TestDurationUtil.java | 92 +
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 53 +-
.../apache/hadoop/ozone/client/OzoneClient.java | 8 +-
.../hadoop/ozone/client/OzoneClientException.java | 8 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 29 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 7 +-
.../org/apache/hadoop/ozone/client/TenantArgs.java | 9 +-
.../client/checksum/BaseFileChecksumHelper.java | 95 +-
.../client/checksum/ECBlockChecksumComputer.java | 72 +-
.../client/checksum/ECFileChecksumHelper.java | 77 +-
.../checksum/ReplicatedFileChecksumHelper.java | 79 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 8 +-
.../ozone/client/io/KeyOutputStreamSemaphore.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 68 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 212 +-
.../TestBlockOutputStreamIncrementalPutBlock.java | 6 +
.../hadoop/ozone/client/TestOzoneClient.java | 2 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 4 +-
.../client/checksum/TestFileChecksumHelper.java | 361 ++
.../checksum/TestReplicatedFileChecksumHelper.java | 370 ---
.../client/io/TestECBlockOutputStreamEntry.java | 2 +-
hadoop-ozone/common/pom.xml | 4 +-
.../apache/hadoop/fs/ozone/OzoneTrashPolicy.java | 208 ++
.../org/apache/hadoop/fs/ozone/package-info.java | 20 +
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 15 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 21 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 11 +
.../ozone/client/io/SelectorOutputStream.java | 2 +-
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 8 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 15 +
.../apache/hadoop/ozone/om/S3SecretManager.java | 2 +-
.../om/exceptions/OMLeaderNotReadyException.java | 2 +-
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 11 +-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 30 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 41 +
.../hadoop/ozone/om/helpers/BasicOmKeyInfo.java | 4 +
.../hadoop/ozone/om/helpers/DeleteTenantState.java | 1 -
.../hadoop/ozone/om/helpers/KeyValueUtil.java | 4 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 3 +-
.../hadoop/ozone/om/helpers/OmDBAccessIdInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmDBTenantState.java | 2 +-
.../ozone/om/helpers/OmDBUserPrincipalInfo.java | 11 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 3 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 8 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 3 +-
.../hadoop/ozone/om/helpers/OmRangerSyncArgs.java | 1 -
.../hadoop/ozone/om/helpers/OmTenantArgs.java | 1 -
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 3 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 51 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 33 +
.../ozone/om/helpers/OzoneIdentityProvider.java | 3 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 9 +-
.../hadoop/ozone/om/helpers/S3SecretValue.java | 3 +-
.../hadoop/ozone/om/helpers/S3VolumeContext.java | 1 -
.../hadoop/ozone/om/helpers/ServiceInfo.java | 46 +-
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 5 +
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 65 +-
.../ozone/om/multitenant/AccountNameSpace.java | 4 +-
.../ozone/om/multitenant/BucketNameSpace.java | 4 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 75 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 1 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 229 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 12 +-
.../ozone/security/OzoneTokenIdentifier.java | 100 +-
.../ozone/security/acl/OzoneAccessAuthorizer.java | 9 +-
.../hadoop/ozone/security/acl/OzoneAclConfig.java | 28 +-
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +-
.../ozone/snapshot/SnapshotDiffReportOzone.java | 1 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 -
.../main/resources/ozone-version-info.properties | 3 -
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 4 +-
.../hadoop/ozone/om/helpers/TestOzoneFsUtils.java | 30 +
hadoop-ozone/csi/pom.xml | 5 +-
hadoop-ozone/datanode/pom.xml | 5 +-
hadoop-ozone/dev-support/checks/_build.sh | 31 +
hadoop-ozone/dev-support/checks/_diffoscope.sh | 45 +
hadoop-ozone/dev-support/checks/_lib.sh | 6 +-
.../dev-support/checks/_mvn_unit_report.sh | 38 +-
hadoop-ozone/dev-support/checks/_post_process.sh | 50 +
hadoop-ozone/dev-support/checks/acceptance.sh | 17 +-
hadoop-ozone/dev-support/checks/author.sh | 14 +-
hadoop-ozone/dev-support/checks/bats.sh | 8 +-
hadoop-ozone/dev-support/checks/build.sh | 18 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 13 +-
hadoop-ozone/dev-support/checks/coverage.sh | 3 +-
hadoop-ozone/dev-support/checks/dependency.sh | 4 +-
hadoop-ozone/dev-support/checks/docs.sh | 9 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 23 +-
hadoop-ozone/dev-support/checks/junit.sh | 12 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 9 +-
hadoop-ozone/dev-support/checks/license.sh | 10 +-
hadoop-ozone/dev-support/checks/rat.sh | 8 +-
hadoop-ozone/dev-support/checks/repro.sh | 37 +
hadoop-ozone/dev-support/checks/sonar.sh | 2 +-
.../dev-support/k8s/regenerate-examples.sh | 27 +
.../dist/dev-support/bin/dist-layout-stitching | 17 +-
hadoop-ozone/dist/pom.xml | 66 +-
.../dist/src/main/assemblies/ozone-src.xml | 49 +-
.../dashboards/Ozone - ReadKey Metrics.json | 3194 ++++++++++--------
.../src/main/compose/common/hadoop-secure.yaml | 6 +-
.../dist/src/main/compose/common/hadoop-test.sh | 28 +-
.../dist/src/main/compose/common/hadoop.conf | 3 -
.../dist/src/main/compose/common/hadoop.yaml | 4 +-
.../dist/src/main/compose/common/init-kdc.sh | 87 +
.../dist/src/main/compose/common/s3a-test.sh | 8 +-
.../src/main/compose/compatibility/docker-config | 3 +-
.../dist/src/main/compose/compatibility/test.sh | 2 +-
.../compose/ozone-balancer/docker-compose.yaml | 46 +
.../src/main/compose/ozone-balancer/docker-config | 6 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 31 +
.../src/main/compose/ozone-balancer/test-ratis.sh | 35 +
.../dist/src/main/compose/ozone-balancer/test.sh | 33 -
.../dist/src/main/compose/ozone-csi/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 +
.../dist/src/main/compose/ozone-ha/docker-config | 2 +-
.../src/main/compose/ozone-om-ha/docker-config | 2 +-
.../main/compose/ozone-om-prepare/docker-config | 2 +-
.../src/main/compose/ozone-topology/docker-config | 2 +-
.../dist/src/main/compose/ozone/docker-config | 2 +-
.../src/main/compose/ozoneblockade/docker-config | 2 +-
.../src/main/compose/ozonescripts/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 4 +-
.../src/main/compose/ozonesecure-ha/docker-config | 3 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +
.../compose/ozonesecure-mr/docker-compose.yaml | 4 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 6 +-
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test-fcq.sh | 1 +
.../dist/src/main/compose/restart/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 52 +-
.../dist/src/main/compose/upgrade/compose/ha/.env | 3 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 6 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 +-
.../src/main/compose/upgrade/compose/non-ha/.env | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 2 +-
.../compose/upgrade/compose/non-ha/docker-config | 3 +-
.../src/main/compose/upgrade/compose/om-ha/.env | 2 +-
.../upgrade/compose/om-ha/docker-compose.yaml | 2 +-
.../compose/upgrade/compose/om-ha/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 5 +-
.../dist/src/main/compose/upgrade/testlib.sh | 4 +-
.../callbacks/1.5.0/callback.sh | 31 +
hadoop-ozone/dist/src/main/compose/xcompat/.env | 5 +
.../dist/src/main/compose/xcompat/clients.yaml | 82 +-
.../dist/src/main/compose/xcompat/docker-config | 96 +-
.../dist/src/main/compose/xcompat/krb5.conf | 41 +
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 114 +
.../dist/src/main/compose/xcompat/new-cluster.yaml | 33 +-
.../dist/src/main/compose/xcompat/old-cluster.yaml | 33 +-
.../dist/src/main/compose/xcompat/test-new.sh | 27 +
.../dist/src/main/compose/xcompat/test-old.sh | 30 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 129 -
.../src/main/k8s/definitions/ozone/config.yaml | 18 +-
.../k8s/definitions/ozone/httpfs-ss-service.yaml | 27 +
.../src/main/k8s/definitions/ozone/httpfs-ss.yaml | 44 +
.../k8s/definitions/ozone/recon-ss-service.yaml | 27 +
.../src/main/k8s/definitions/ozone/recon-ss.yaml | 52 +
hadoop-ozone/dist/src/main/k8s/examples/README.md | 22 +
.../examples/getting-started/config-configmap.yaml | 16 +-
.../getting-started/httpfs-public-service.yaml | 28 +
.../examples/getting-started/httpfs-service.yaml | 28 +
.../getting-started/httpfs-statefulset.yaml | 55 +
.../examples/getting-started/kustomization.yaml | 6 +
.../getting-started/recon-public-service.yaml | 28 +
.../examples/getting-started/recon-service.yaml | 28 +
.../getting-started/recon-statefulset.yaml | 63 +
.../k8s/examples/minikube/config-configmap.yaml | 16 +-
.../examples/minikube/httpfs-public-service.yaml | 28 +
.../main/k8s/examples/minikube/httpfs-service.yaml | 28 +
.../k8s/examples/minikube/httpfs-statefulset.yaml | 55 +
.../main/k8s/examples/minikube/kustomization.yaml | 6 +
.../examples/minikube/recon-public-service.yaml | 28 +
.../main/k8s/examples/minikube/recon-service.yaml | 28 +
.../k8s/examples/minikube/recon-statefulset.yaml | 63 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 16 +-
.../examples/ozone-dev/httpfs-public-service.yaml | 28 +
.../k8s/examples/ozone-dev/httpfs-service.yaml | 28 +
.../k8s/examples/ozone-dev/httpfs-statefulset.yaml | 62 +
.../main/k8s/examples/ozone-dev/kustomization.yaml | 6 +
.../examples/ozone-dev/recon-public-service.yaml | 28 +
.../main/k8s/examples/ozone-dev/recon-service.yaml | 28 +
.../k8s/examples/ozone-dev/recon-statefulset.yaml | 69 +
.../k8s/examples/ozone-ha/config-configmap.yaml | 14 +-
.../main/k8s/examples/ozone-ha/httpfs-service.yaml | 28 +
.../k8s/examples/ozone-ha/httpfs-statefulset.yaml | 61 +
.../main/k8s/examples/ozone-ha/kustomization.yaml | 4 +
.../main/k8s/examples/ozone-ha/recon-service.yaml | 28 +
.../k8s/examples/ozone-ha/recon-statefulset.yaml | 70 +
.../main/k8s/examples/ozone/config-configmap.yaml | 16 +-
.../main/k8s/examples/ozone/httpfs-service.yaml | 28 +
.../k8s/examples/ozone/httpfs-statefulset.yaml | 61 +
.../src/main/k8s/examples/ozone/kustomization.yaml | 4 +
.../src/main/k8s/examples/ozone/recon-service.yaml | 28 +
.../main/k8s/examples/ozone/recon-statefulset.yaml | 70 +
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 580 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 576 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 146 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/httpfs.keytab | Bin 616 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 142 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 138 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 576 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 606 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 138 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 586 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 586 -> 0 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 908 -> 0 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 920 -> 0 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 98 -
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 6 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../src/main/license/src/licenses/IMPORTANT.md | 6 +-
.../src/main/smoketest/admincli/container.robot | 12 +
.../dist/src/main/smoketest/admincli/scmrole.robot | 6 +-
.../src/main/smoketest/balancer/testBalancer.robot | 69 +-
.../dist/src/main/smoketest/compatibility/om.robot | 5 +
.../src/main/smoketest/compatibility/read.robot | 137 +-
.../src/main/smoketest/compatibility/setup.robot | 11 +-
.../src/main/smoketest/compatibility/write.robot | 56 +-
.../src/main/smoketest/debug/ozone-debug-ldb.robot | 146 +
.../debug/ozone-debug-lease-recovery.robot | 22 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 5 +
.../src/main/smoketest/ec/backward-compat.robot | 107 -
.../dist/src/main/smoketest/ec/lib.resource | 12 -
.../dist/src/main/smoketest/freon/hsync.robot | 51 +
.../main/smoketest/hsync/upgrade-hsync-check.robot | 68 +
.../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +-
.../dist/src/main/smoketest/omha/om-roles.robot | 15 +
.../dist/src/main/smoketest/ozone-lib/freon.robot | 6 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 14 +-
.../src/main/smoketest/ozone-lib/shell_tests.robot | 8 +
.../dist/src/main/smoketest/recon/recon-api.robot | 12 +
.../src/main/smoketest/s3/MultipartUpload.robot | 191 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 9 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 13 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 31 +-
.../dist/src/main/smoketest/s3/mpu_lib.robot | 105 +
.../dist/src/main/smoketest/s3/objectcopys3a.robot | 8 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 1 +
.../dist/src/main/smoketest/s3/objecttagging.robot | 73 +
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../src/main/smoketest/s3/secretgenerate.robot | 8 +-
.../dist/src/main/smoketest/s3/secretrevoke.robot | 8 +-
.../smoketest/security/bucket-encryption.robot | 45 -
hadoop-ozone/dist/src/shell/conf/log4j.properties | 9 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 5 +
.../dist/src/shell/ozone/ozone-functions.sh | 8 -
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/httpfsgateway/pom.xml | 5 +-
.../src/main/resources/httpfs.properties | 3 -
hadoop-ozone/insight/pom.xml | 6 +-
.../org/apache/hadoop/ozone/insight/Insight.java | 4 -
hadoop-ozone/integration-test/pom.xml | 21 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 69 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 10 +-
.../hadoop/fs/ozone/OzoneFileSystemTests.java | 18 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 140 +
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 134 +-
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 2 +
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 2 +
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 12 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 49 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 18 +
.../ozone/contract/AbstractOzoneContractTest.java | 3 +
.../apache/hadoop/hdds/scm/TestCommitInRatis.java | 189 ++
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 91 -
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 9 +-
.../hdds/scm/TestSCMInstallSnapshotWithHA.java | 5 +-
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 20 +-
.../hdds/scm/TestStorageContainerManager.java | 16 +-
.../hdds/scm/TestStorageContainerManagerHA.java | 83 +-
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 408 +++
.../hadoop/hdds/scm/TestXceiverClientManager.java | 22 +-
.../hadoop/hdds/scm/TestXceiverClientMetrics.java | 11 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 6 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 106 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 38 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 133 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 8 +-
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 11 +-
.../hadoop/ozone/TestContainerOperations.java | 19 +
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 44 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 5 +-
.../hadoop/ozone/TestMultipartObjectGet.java | 4 +-
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 15 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 32 +-
.../client/rpc/AbstractTestECKeyOutputStream.java | 490 ---
.../ozone/client/rpc/OzoneRpcClientTests.java | 303 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 6 -
.../ozone/client/rpc/TestBlockOutputStream.java | 9 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 19 +-
.../hadoop/ozone/client/rpc/TestCommitInRatis.java | 193 --
.../rpc/TestContainerReplicationEndToEnd.java | 7 +-
.../client/rpc/TestContainerStateMachine.java | 6 -
.../TestContainerStateMachineFailureOnRead.java | 5 -
.../rpc/TestContainerStateMachineFailures.java | 55 +
.../rpc/TestContainerStateMachineFlushDelay.java | 6 -
.../client/rpc/TestDeleteWithInAdequateDN.java | 7 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 570 +++-
.../rpc/TestECKeyOutputStreamWithZeroCopy.java | 31 -
.../client/rpc/TestOzoneAtRestEncryption.java | 6 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 7 +
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 9 +-
.../ozone/client/rpc/TestWatchForCommit.java | 414 ---
.../ozone/container/TestContainerReplication.java | 1 -
.../commandhandler/TestBlockDeletion.java | 44 +-
.../transport/server/ratis/TestCSMMetrics.java | 27 +-
.../container/metrics/TestContainerMetrics.java | 25 +-
.../container/ozoneimpl/TestOzoneContainer.java | 169 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 10 +-
.../replication/TestContainerReplication.java | 2 +-
.../container/server/TestContainerServer.java | 21 +-
.../server/TestSecureContainerServer.java | 23 +-
.../org/apache/hadoop/ozone/debug/TestLDBCli.java | 140 +-
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 2 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 29 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 15 +-
.../hadoop/ozone/freon/TestHsyncGenerator.java | 9 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 15 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 15 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 14 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 12 +-
.../ozone/om/TestBucketLayoutWithOlderClient.java | 4 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 2 +
.../hadoop/ozone/om/TestListKeysWithFSO.java | 2 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 16 +-
.../ozone/om/TestOmContainerLocationCache.java | 14 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 39 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 109 +-
.../om/TestOzoneManagerListVolumesSecure.java | 2 +
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 79 +
.../ozone/om/service/TestRangerBGSyncService.java | 8 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 173 +-
.../om/snapshot/TestOmSnapshotFileSystem.java | 70 +-
.../om/snapshot/TestOmSnapshotFileSystemFso.java | 4 +-
...stOmSnapshotFileSystemFsoWithLinkedBuckets.java | 31 +
.../snapshot/TestOmSnapshotFileSystemLegacy.java | 4 +-
...mSnapshotFileSystemLegacyWithLinkedBuckets.java | 31 +
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 +-
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 36 +
.../TestOmSnapshotFsoWithoutNativeLib.java | 2 +-
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 34 +
.../ozone/om/snapshot/TestOmSnapshotLegacy.java | 34 -
.../om/snapshot/TestOmSnapshotObjectStore.java | 2 +-
...TestOmSnapshotObjectStoreWithLinkedBuckets.java | 34 +
.../TestOmSnapshotWithBucketLinkingLegacy.java | 34 +
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 34 +
.../om/snapshot/TestOzoneManagerHASnapshot.java | 104 +
.../snapshot/TestSnapshotBackgroundServices.java | 2 +
.../om/snapshot/TestSnapshotDeletingService.java | 595 ----
...TestSnapshotDeletingServiceIntegrationTest.java | 919 ++++++
.../TestSnapshotDirectoryCleaningService.java | 2 +
.../ozone/parser/TestOzoneHARatisLogParser.java | 4 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +
.../apache/hadoop/ozone/recon/TestReconTasks.java | 44 +-
.../hadoop/ozone/repair/om/TestFSORepairTool.java | 541 +++
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 902 +++++
.../hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java | 47 +
.../ozone/s3/awssdk/v1/TestS3SDKV1WithRatis.java | 49 +
.../awssdk/v1/TestS3SDKV1WithRatisStreaming.java | 55 +
.../ozone/shell/TestDeletedBlocksTxnShell.java | 22 +
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 6 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 10 +-
.../hadoop/ozone/shell/TestOzoneRepairShell.java | 36 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 65 +-
.../ozone/shell/TestOzoneShellHAWithFSO.java | 2 +
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 15 +-
.../hadoop/ozone/shell/TestReconfigShell.java | 4 +-
.../ozone/shell/TestTransferLeadershipShell.java | 16 +-
.../tools/contract/AbstractContractDistCpTest.java | 47 +-
.../org/apache/hadoop/utils/InputSubstream.java | 115 +
.../src/test/resources/log4j.properties | 1 +
.../src/test/resources/ozone-site.xml | 2 +-
hadoop-ozone/interface-client/pom.xml | 14 +-
.../src/main/proto/OmClientProtocol.proto | 123 +-
.../interface-client/src/main/resources/proto.lock | 136 +-
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 48 +-
.../ozone/om/codec/TokenIdentifierCodec.java | 15 +-
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 3 +-
hadoop-ozone/ozone-manager/pom.xml | 8 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 6 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 80 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 155 +-
.../org/apache/hadoop/ozone/om/ListIterator.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 6 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 58 +-
.../hadoop/ozone/om/OMMultiTenantManager.java | 2 +-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 1 -
.../hadoop/ozone/om/OMPerformanceMetrics.java | 20 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 203 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 40 +
.../hadoop/ozone/om/OmMetadataReaderMetrics.java | 4 +
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 6 +
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 86 +-
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 17 +
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 42 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 224 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 12 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 62 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 31 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 168 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 62 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 3 +
.../ozone/om/multitenant/AuthorizerLock.java | 2 +-
.../RangerClientMultiTenantAccessController.java | 12 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 40 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 69 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 43 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 36 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 21 +
.../hadoop/ozone/om/request/OMClientRequest.java | 7 +-
.../ozone/om/request/OMClientRequestUtils.java | 13 +
.../hadoop/ozone/om/request/RequestAuditor.java | 1 -
.../om/request/bucket/OMBucketCreateRequest.java | 13 +-
.../om/request/file/OMDirectoryCreateRequest.java | 137 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 89 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 22 +-
.../request/file/OMFileCreateRequestWithFSO.java | 7 +-
.../ozone/om/request/file/OMFileRequest.java | 2 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 50 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 26 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 25 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 74 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 13 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 4 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 331 +-
.../S3InitiateMultipartUploadRequest.java | 2 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 6 +-
.../s3/tagging/S3DeleteObjectTaggingRequest.java | 190 ++
.../S3DeleteObjectTaggingRequestWithFSO.java | 169 +
.../s3/tagging/S3PutObjectTaggingRequest.java | 192 ++
.../tagging/S3PutObjectTaggingRequestWithFSO.java | 171 +
.../ozone/om/request/s3/tagging/package-info.java | 21 +
.../request/snapshot/OMSnapshotCreateRequest.java | 27 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 7 +
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 32 +-
.../snapshot/OMSnapshotMoveTableKeysRequest.java | 184 ++
.../request/snapshot/OMSnapshotPurgeRequest.java | 96 +-
.../request/snapshot/OMSnapshotRenameRequest.java | 8 +
.../om/request/volume/OMVolumeCreateRequest.java | 17 +
.../ozone/om/request/volume/OMVolumeRequest.java | 1 -
.../key/OMDirectoriesPurgeResponseWithFSO.java | 13 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 15 +-
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 73 +
.../S3DeleteObjectTaggingResponseWithFSO.java | 72 +
.../s3/tagging/S3PutObjectTaggingResponse.java | 72 +
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 72 +
.../ozone/om/response/s3/tagging/package-info.java | 21 +
.../OMSnapshotMoveDeletedKeysResponse.java | 50 +-
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 162 +
.../response/snapshot/OMSnapshotPurgeResponse.java | 15 +-
.../om/service/AbstractKeyDeletingService.java | 106 +-
.../ozone/om/service/DirectoryDeletingService.java | 267 +-
.../ozone/om/service/KeyDeletingService.java | 89 +-
.../om/service/MultipartUploadCleanupService.java | 24 +-
.../ozone/om/service/OMRangerBGSyncService.java | 23 +-
.../ozone/om/service/OpenKeyCleanupService.java | 74 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 106 +-
.../ozone/om/service/SnapshotDeletingService.java | 550 +--
.../service/SnapshotDirectoryCleaningService.java | 36 +-
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 2 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 47 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 157 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 26 +-
.../protocolPB/OzoneManagerRequestHandler.java | 101 +-
.../hadoop/ozone/protocolPB/RequestHandler.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 77 +-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 8 +-
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 45 +-
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 3 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 25 +
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 44 -
.../apache/hadoop/ozone/om/TestTrashService.java | 137 -
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 63 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 13 +-
.../ozone/om/request/OMRequestTestUtils.java | 81 +-
.../request/TestBucketLayoutAwareOMKeyFactory.java | 6 +-
.../request/bucket/TestOMBucketCreateRequest.java | 7 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 7 +-
.../request/file/TestOMDirectoryCreateRequest.java | 13 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 15 +-
.../om/request/file/TestOMFileCreateRequest.java | 15 +-
.../file/TestOMFileCreateRequestWithFSO.java | 8 +-
.../TestOMDirectoriesPurgeRequestAndResponse.java | 91 +-
.../om/request/key/TestOMKeyCommitRequest.java | 20 +-
.../om/request/key/TestOMKeyCreateRequest.java | 25 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 13 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 64 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 62 +-
.../TestS3ExpiredMultipartUploadsAbortRequest.java | 2 +
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 9 +-
.../s3/multipart/TestS3MultipartRequest.java | 20 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +-
.../TestS3MultipartUploadCommitPartRequest.java | 2 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 13 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 19 +-
.../tagging/TestS3DeleteObjectTaggingRequest.java | 202 ++
.../TestS3DeleteObjectTaggingRequestWithFSO.java | 74 +
.../s3/tagging/TestS3PutObjectTaggingRequest.java | 254 ++
.../TestS3PutObjectTaggingRequestWithFSO.java | 95 +
.../snapshot/TestOMSnapshotCreateRequest.java | 202 +-
.../snapshot/TestOMSnapshotDeleteRequest.java | 184 +-
.../TestOMSnapshotMoveTableKeysRequest.java | 264 ++
.../TestOMSnapshotPurgeRequestAndResponse.java | 225 +-
.../snapshot/TestOMSnapshotRenameRequest.java | 202 +-
...estOMSnapshotSetPropertyRequestAndResponse.java | 86 +-
.../om/request/volume/TestOMVolumeRequest.java | 1 +
.../tagging/TestS3DeleteObjectTaggingResponse.java | 89 +
.../TestS3DeleteObjectTaggingResponseWithFSO.java | 74 +
.../s3/tagging/TestS3PutObjectTaggingResponse.java | 88 +
.../TestS3PutObjectTaggingResponseWithFSO.java | 70 +
.../snapshot/TestOMSnapshotCreateResponse.java | 4 +-
.../TestOMSnapshotMoveTableKeysResponse.java | 199 ++
.../volume/TestOMVolumeCreateResponse.java | 49 +-
.../volume/TestOMVolumeDeleteResponse.java | 44 +-
.../om/response/volume/TestOMVolumeResponse.java | 64 +
.../volume/TestOMVolumeSetOwnerResponse.java | 57 +-
.../volume/TestOMVolumeSetQuotaResponse.java | 42 +-
.../om/service/TestDirectoryDeletingService.java | 60 +
.../ozone/om/service/TestKeyDeletingService.java | 162 +-
.../om/service/TestOpenKeyCleanupService.java | 26 +-
.../ozone/om/service/TestQuotaRepairTask.java | 23 +-
.../om/service/TestSnapshotDeletingService.java | 38 +-
.../ozone/om/snapshot/TestSnapshotChain.java | 32 +
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 61 +-
.../hadoop/ozone/om/snapshot/TestSnapshotInfo.java | 47 +
.../snapshot/TestSnapshotRequestAndResponse.java | 261 ++
.../protocolPB/TestOzoneManagerRequestHandler.java | 175 +
.../hadoop/ozone/security/SecretKeyTestClient.java | 73 +
.../TestOzoneDelegationTokenSecretManager.java | 63 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 3 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 3 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 6 +-
hadoop-ozone/ozonefs-common/pom.xml | 29 +-
.../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 9 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 93 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 45 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 235 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 52 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 2 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 6 +
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 1 -
.../apache/hadoop/fs/ozone/TestOzoneFsShell.java | 9 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 9 +
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 9 +
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 9 +
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 9 +
hadoop-ozone/ozonefs-shaded/pom.xml | 10 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 9 +
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 9 +
hadoop-ozone/pom.xml | 80 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../recon/codegen/ReconSchemaGenerationModule.java | 2 +
.../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 22 +
.../recon/schema/ContainerSchemaDefinition.java | 6 +-
.../recon/schema/SchemaVersionTableDefinition.java | 109 +
hadoop-ozone/recon/pom.xml | 7 +-
.../hadoop/ozone/recon/ConfigurationProvider.java | 2 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 10 +-
.../apache/hadoop/ozone/recon/ReconContext.java | 5 +-
.../hadoop/ozone/recon/ReconResponseUtils.java | 2 +-
.../hadoop/ozone/recon/ReconSchemaManager.java | 51 +-
.../recon/ReconSchemaVersionTableManager.java | 105 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 16 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 302 +-
.../ozone/recon/api/AccessHeatMapEndpoint.java | 5 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 25 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 51 +-
.../ozone/recon/api/OMDBInsightEndpoint.java | 579 ++--
.../ozone/recon/api/OMDBInsightSearchEndpoint.java | 389 ---
.../ozone/recon/api/ServiceNotReadyException.java | 29 +
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 1 +
.../ozone/recon/api/handlers/BucketHandler.java | 2 +-
.../hadoop/ozone/recon/api/types/AclMetadata.java | 17 +-
.../recon/api/types/ClusterStateResponse.java | 1 -
.../ozone/recon/api/types/DatanodeMetadata.java | 48 +-
.../ozone/recon/api/types/KeyEntityInfo.java | 4 +-
.../recon/api/types/KeyEntityInfoProtoWrapper.java | 145 +
.../ozone/recon/api/types/ListKeysResponse.java | 6 +-
.../recon/api/types/NamespaceSummaryResponse.java | 15 +-
.../hadoop/ozone/recon/api/types/ParamInfo.java | 14 +
.../ozone/recon/api/types/PipelineMetadata.java | 1 -
.../api/types/UnhealthyContainersResponse.java | 13 -
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 5 +
.../ozone/recon/fsck/ContainerHealthTask.java | 153 +-
.../persistence/ContainerHealthSchemaManager.java | 4 +-
.../recon/recovery/ReconOMMetadataManager.java | 14 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 9 +
.../recon/scm/ContainerReplicaHistoryList.java | 8 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 11 -
.../ozone/recon/scm/ReconSCMDBDefinition.java | 24 +-
.../scm/ReconStorageContainerManagerFacade.java | 27 +-
.../recon/spi/ReconContainerMetadataManager.java | 16 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 17 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 15 +-
.../impl/ReconContainerMetadataManagerImpl.java | 12 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 12 -
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 6 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 6 +-
.../ozone/recon/tasks/OmUpdateEventValidator.java | 2 -
.../hadoop/ozone/recon/tasks/ReconOmTask.java | 4 +-
.../ozone/recon/tasks/ReconTaskController.java | 2 +-
.../upgrade/InitialConstraintUpgradeAction.java | 114 +
.../ozone/recon/upgrade/ReconLayoutFeature.java | 115 +
.../recon/upgrade/ReconLayoutVersionManager.java | 161 +
.../ozone/recon/upgrade/ReconUpgradeAction.java | 51 +
.../ozone/recon/upgrade/UpgradeActionRecon.java | 67 +
.../hadoop/ozone/recon/upgrade/package-info.java | 29 +
.../webapps/recon/ozone-recon-web/api/db.json | 659 ++--
.../webapps/recon/ozone-recon-web/package.json | 7 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1243 +++----
.../webapps/recon/ozone-recon-web/src/app.less | 5 +
.../webapps/recon/ozone-recon-web/src/app.tsx | 71 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 3 +-
.../src/components/navBar/navBar.less | 2 +-
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../src/utils/axiosRequestHelper.tsx | 6 +-
.../src/v2/components/breadcrumbs/breadcrumbs.tsx | 55 +
.../decommissioningSummary.tsx | 139 +
.../components/duBreadcrumbNav/duBreadcrumbNav.tsx | 175 +
.../src/v2/components/duMetadata/duMetadata.tsx | 389 +++
.../src/v2/components/eChart/eChart.tsx | 15 +-
.../src/v2/components/navBar/navBar.less | 65 +
.../src/v2/components/navBar/navBar.tsx | 180 +
.../overviewCard/overviewSummaryCard.tsx | 10 +-
.../src/v2/components/plots/duPieChart.tsx | 211 ++
.../src/v2/components/plots/heatmapPlot.tsx | 151 +
.../v2/components/plots/insightsContainerPlot.tsx | 149 +
.../src/v2/components/plots/insightsFilePlot.tsx | 251 ++
.../src/v2/components/search/search.tsx | 6 +
.../src/v2/components/select/multiSelect.tsx | 104 +-
.../src/v2/components/select/singleSelect.tsx | 4 +-
.../src/v2/components/storageBar/storageBar.less | 45 +
.../src/v2/components/storageBar/storageBar.tsx | 49 +-
.../src/v2/components/tables/bucketsTable.tsx | 267 ++
.../src/v2/components/tables/containersTable.tsx | 259 ++
.../src/v2/components/tables/datanodesTable.tsx | 314 ++
.../tables/insights/containerMismatchTable.tsx | 215 ++
.../tables/insights/deletePendingDirsTable.tsx | 142 +
.../tables/insights/deletePendingKeysTable.tsx | 194 ++
.../tables/insights/deletedContainerKeysTable.tsx | 163 +
.../tables/insights/expandedKeyTable.tsx | 93 +
.../tables/insights/expandedPendingKeysTable.tsx | 81 +
.../components/tables/insights/openKeysTable.tsx | 213 ++
.../src/v2/components/tables/pipelinesTable.tsx | 211 ++
.../src/v2/components/tables/volumesTable.tsx | 178 +
.../src/v2/constants/breadcrumbs.constants.tsx | 34 +
.../src/v2/constants/heatmap.constants.tsx | 47 +
.../src/v2/constants/limit.constants.tsx | 37 +
.../src/v2/pages/buckets/buckets.less | 41 +
.../src/v2/pages/buckets/buckets.tsx | 326 ++
.../src/v2/pages/containers/containers.less | 50 +
.../src/v2/pages/containers/containers.tsx | 283 ++
.../src/v2/pages/datanodes/datanodes.less | 52 +
.../src/v2/pages/datanodes/datanodes.tsx | 309 ++
.../src/v2/pages/diskUsage/diskUsage.less | 68 +
.../src/v2/pages/diskUsage/diskUsage.tsx | 144 +
.../src/v2/pages/heatmap/heatmap.less | 86 +
.../src/v2/pages/heatmap/heatmap.tsx | 388 +++
.../src/v2/pages/insights/insights.less | 36 +
.../src/v2/pages/insights/insights.tsx | 196 ++
.../src/v2/pages/insights/omInsights.tsx | 208 ++
.../src/v2/pages/notFound/notFound.tsx | 103 +
.../src/v2/pages/overview/overview.tsx | 64 +-
.../src/v2/pages/pipelines/pipelines.less | 48 +
.../src/v2/pages/pipelines/pipelines.tsx | 160 +
.../src/v2/pages/volumes/volumes.tsx | 179 +-
.../recon/ozone-recon-web/src/v2/routes-v2.tsx | 41 +
.../ozone-recon-web/src/v2/types/bucket.types.ts | 28 +-
.../src/v2/types/container.types.ts | 94 +
.../ozone-recon-web/src/v2/types/datanode.types.ts | 167 +
.../src/v2/types/diskUsage.types.ts | 41 +
.../ozone-recon-web/src/v2/types/heatmap.types.ts | 51 +
.../ozone-recon-web/src/v2/types/insights.types.ts | 199 ++
.../src/v2/types/pipelines.types.ts | 62 +
.../ozone-recon-web/src/v2/types/volume.types.ts | 10 +-
.../ozone-recon-web/src/v2/utils/momentUtils.ts | 68 +
.../src/views/datanodes/datanodes.tsx | 12 -
.../src/views/insights/insights.tsx | 8 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 20 +-
.../webapps/recon/ozone-recon-web/vite.config.ts | 9 +-
.../ozone/recon/api/TestContainerEndpoint.java | 47 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 549 +++
.../hadoop/ozone/recon/api/TestEndpoints.java | 3 -
.../recon/api/TestNSSummaryEndpointWithFSO.java | 13 +-
.../recon/api/TestOMDBInsightSearchEndpoint.java | 796 -----
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 219 +-
.../ozone/recon/api/TestOpenContainerCount.java | 1 -
.../recon/api/TestOpenKeysSearchEndpoint.java | 838 +++++
.../ozone/recon/api/filters/TestAdminFilter.java | 12 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 283 +-
.../TestContainerHealthTaskRecordGenerator.java | 62 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 4 +
.../TestSchemaVersionTableDefinition.java | 311 ++
.../scm/AbstractReconContainerManagerTest.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 2 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 2 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 2 +-
.../recon/tasks/TestOmUpdateEventValidator.java | 5 +-
.../TestInitialConstraintUpgradeAction.java | 192 ++
.../upgrade/TestReconLayoutVersionManager.java | 365 ++
hadoop-ozone/s3-secret-store/pom.xml | 4 +-
hadoop-ozone/s3gateway/pom.xml | 5 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 6 +-
.../hadoop/ozone/s3/AuthorizationFilter.java | 9 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 31 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 2 -
.../hadoop/ozone/s3/OzoneConfigurationHolder.java | 20 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 2 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 6 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 7 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 50 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 68 +-
.../ozone/s3/endpoint/MessageUnmarshaller.java | 95 +
.../endpoint/MultiDeleteRequestUnmarshaller.java | 52 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 153 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 6 +-
.../endpoint/PutBucketAclRequestUnmarshaller.java | 57 +-
.../ozone/s3/endpoint/PutTaggingUnmarshaller.java | 29 +
.../apache/hadoop/ozone/s3/endpoint/S3Tagging.java | 155 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 5 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 128 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 2 +-
.../hadoop/ozone/s3/signature/Credential.java | 3 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 1 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 9 +
.../hadoop/ozone/s3secret/S3AdminEndpoint.java | 34 +
.../hadoop/ozone/s3secret/S3SecretAdminFilter.java | 60 +
.../ozone/s3secret/S3SecretManagementEndpoint.java | 8 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 32 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 31 +
.../s3/endpoint/TestAbortMultipartUpload.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 14 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 2 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 6 +-
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 22 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 85 +-
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 152 +
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 125 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 264 ++
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 17 +-
.../s3/endpoint/TestPartUploadWithStream.java | 8 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 48 +-
.../ozone/s3/endpoint/TestUploadWithStream.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 148 +-
.../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +-
.../hadoop/ozone/s3secret/TestSecretRevoke.java | 2 -
hadoop-ozone/tools/pom.xml | 39 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 22 +-
.../ozone/admin/om/CancelPrepareSubCommand.java | 2 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 39 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 23 +-
.../ozone/admin/om/UpdateRangerSubcommand.java | 4 -
.../reconfig/AbstractReconfigureSubCommand.java | 1 +
.../ozone/admin/reconfig/ReconfigureCommands.java | 12 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 35 +-
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 24 +-
.../hadoop/ozone/debug/ChunkDataNodeDetails.java | 43 -
.../apache/hadoop/ozone/debug/ChunkDetails.java | 54 -
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 210 --
.../org/apache/hadoop/ozone/debug/ChunkType.java | 24 -
.../ozone/debug/CompactionLogDagPrinter.java | 14 +-
.../hadoop/ozone/debug/ContainerChunkInfo.java | 108 -
.../hadoop/ozone/debug/DBDefinitionFactory.java | 32 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 672 ----
.../org/apache/hadoop/ozone/debug/DropTable.java | 81 -
.../hadoop/ozone/debug/FindMissingPadding.java | 14 +-
.../apache/hadoop/ozone/debug/LeaseRecoverer.java | 14 +-
.../org/apache/hadoop/ozone/debug/ListTables.java | 59 -
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 37 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 11 +-
.../org/apache/hadoop/ozone/debug/RDBParser.java | 66 -
.../apache/hadoop/ozone/debug/ReadReplicas.java | 14 +-
.../org/apache/hadoop/ozone/debug/ValueSchema.java | 185 --
.../apache/hadoop/ozone/debug/VersionDebug.java | 72 +
.../ozone/debug/chunk/ChunkDataNodeDetails.java | 43 +
.../hadoop/ozone/debug/chunk/ChunkDetails.java | 54 +
.../hadoop/ozone/debug/chunk/ChunkKeyHandler.java | 204 ++
.../apache/hadoop/ozone/debug/chunk/ChunkType.java | 24 +
.../ozone/debug/chunk/ContainerChunkInfo.java | 108 +
.../hadoop/ozone/debug/chunk/package-info.java | 22 +
.../ozone/debug/container/ContainerCommands.java | 13 +-
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 987 ++++++
.../apache/hadoop/ozone/debug/ldb/DropTable.java | 76 +
.../apache/hadoop/ozone/debug/ldb/ListTables.java | 50 +
.../apache/hadoop/ozone/debug/ldb/RDBParser.java | 67 +
.../apache/hadoop/ozone/debug/ldb/ValueSchema.java | 179 +
.../hadoop/ozone/debug/ldb/package-info.java | 22 +
.../ozone/debug/segmentparser/BaseLogParser.java | 56 +
.../segmentparser/DatanodeRatisLogParser.java | 60 +
.../debug/segmentparser/GenericRatisLogParser.java | 43 +
.../debug/segmentparser/OMRatisLogParser.java | 45 +
.../ozone/debug/segmentparser/RatisLogParser.java | 42 +
.../debug/segmentparser/SCMRatisLogParser.java | 44 +
.../ozone/debug/segmentparser/package-info.java | 22 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +
.../ozone/freon/ClosedContainerReplicator.java | 21 +-
.../hadoop/ozone/freon/ContentGenerator.java | 15 +-
.../hadoop/ozone/freon/DNRPCLoadGenerator.java | 10 +-
.../hadoop/ozone/freon/DatanodeSimulator.java | 7 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 7 +-
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/HsyncGenerator.java | 128 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 5 +
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 3 +-
.../hadoop/ozone/freon/OzoneClientCreator.java | 65 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 18 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 6 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 9 +-
.../freon/containergenerator/GeneratorScm.java | 2 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 4 +-
.../apache/hadoop/ozone/repair/OzoneRepair.java | 38 +-
.../org/apache/hadoop/ozone/repair/RDBRepair.java | 58 -
.../hadoop/ozone/repair/RecoverSCMCertificate.java | 256 ++
.../hadoop/ozone/repair/TransactionInfoRepair.java | 135 -
.../apache/hadoop/ozone/repair/ldb/RDBRepair.java | 57 +
.../hadoop/ozone/repair/ldb/SnapshotRepair.java | 177 +
.../ozone/repair/ldb/TransactionInfoRepair.java | 125 +
.../hadoop/ozone/repair/ldb/package-info.java | 22 +
.../hadoop/ozone/repair/om/FSORepairCLI.java | 78 +
.../hadoop/ozone/repair/om/FSORepairTool.java | 710 ++++
.../apache/hadoop/ozone/repair/om/OMRepair.java | 47 +
.../hadoop/ozone/repair/om/SnapshotRepair.java | 186 --
.../hadoop/ozone/repair/quota/QuotaRepair.java | 120 +
.../hadoop/ozone/repair/quota/QuotaStatus.java | 71 +
.../hadoop/ozone/repair/quota/QuotaTrigger.java | 92 +
.../hadoop/ozone/repair/quota/package-info.java | 22 +
.../hadoop/ozone/segmentparser/BaseLogParser.java | 56 -
.../segmentparser/DatanodeRatisLogParser.java | 60 -
.../ozone/segmentparser/GenericRatisLogParser.java | 43 -
.../ozone/segmentparser/OMRatisLogParser.java | 45 -
.../hadoop/ozone/segmentparser/RatisLogParser.java | 53 -
.../ozone/segmentparser/SCMRatisLogParser.java | 44 -
.../hadoop/ozone/segmentparser/package-info.java | 22 -
.../org/apache/hadoop/ozone/shell/Handler.java | 6 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 10 +-
.../org/apache/hadoop/ozone/shell/OzoneRatis.java | 51 +
.../org/apache/hadoop/ozone/shell/OzoneShell.java | 20 +-
.../java/org/apache/hadoop/ozone/shell/REPL.java | 90 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 43 +-
.../apache/hadoop/ozone/shell/StoreTypeOption.java | 2 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 12 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 8 +-
.../ozone/shell/bucket/SetEncryptionKey.java | 3 +-
.../shell/bucket/SetReplicationConfigHandler.java | 3 +-
.../ozone/shell/bucket/UpdateBucketHandler.java | 3 +-
.../ozone/shell/checknative/CheckNative.java | 22 +-
.../hadoop/ozone/shell/keys/CatKeyHandler.java | 3 +-
.../ozone/shell/keys/ChecksumKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 2 +-
.../hadoop/ozone/shell/keys/KeyCommands.java | 11 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 16 +-
.../hadoop/ozone/shell/keys/RenameKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/RewriteKeyHandler.java | 3 +-
.../hadoop/ozone/shell/prefix/PrefixCommands.java | 12 +-
.../apache/hadoop/ozone/shell/s3/S3Handler.java | 2 +-
.../shell/snapshot/RenameSnapshotHandler.java | 3 +-
.../ozone/shell/snapshot/SnapshotCommands.java | 12 +-
.../hadoop/ozone/shell/tenant/TenantHandler.java | 2 +-
.../ozone/shell/tenant/TenantUserCommands.java | 11 +-
.../ozone/shell/token/CancelTokenHandler.java | 3 +-
.../hadoop/ozone/shell/token/GetTokenHandler.java | 2 +-
.../ozone/shell/token/RenewTokenHandler.java | 3 +-
.../hadoop/ozone/shell/token/TokenCommands.java | 11 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +-
.../hadoop/ozone/shell/volume/VolumeCommands.java | 12 +-
.../java/org/apache/hadoop/ozone/utils/Filter.java | 110 +
.../hadoop/ozone/utils/FormattingCLIUtils.java | 291 ++
.../hadoop/ozone/checknative/TestCheckNative.java | 18 +
.../ozone/debug/TestDBDefinitionFactory.java | 27 +-
.../TestGenerateOzoneRequiredConfigurations.java | 16 +-
.../ozone/repair/TestTransactionInfoRepair.java | 146 -
.../repair/ldb/TestTransactionInfoRepair.java | 146 +
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 87 +
.../shell/TestOzoneAddressClientCreation.java | 24 +-
.../apache/hadoop/ozone/shell/TestOzoneRatis.java | 172 +
.../ozone/shell/keys/TestChecksumKeyHandler.java | 3 +-
pom.xml | 1965 +++++------
1390 files changed, 52894 insertions(+), 27388 deletions(-)
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
index 969add4a15,0000000000..3c08e58f9b
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
@@@ -1,141 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.checksum;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.client.ClientTrustManager;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.hdds.security.SecurityConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
- import org.apache.hadoop.hdds.security.x509.certificate.client.CACertificateProvider;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
- import org.apache.hadoop.hdds.utils.HAUtils;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import jakarta.annotation.Nonnull;
+import org.apache.hadoop.ozone.container.common.helpers.TokenHelper;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.ozone.container.common.helpers.TokenHelper.encode;
+
+/**
+ * This class wraps necessary container-level rpc calls for container reconciliation.
+ * - GetContainerMerkleTree
+ */
+public class DNContainerOperationClient implements AutoCloseable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DNContainerOperationClient.class);
+ private final TokenHelper tokenHelper;
+ private final XceiverClientManager xceiverClientManager;
+
+ public DNContainerOperationClient(ConfigurationSource conf,
+ CertificateClient certificateClient,
+ SecretKeySignerClient secretKeyClient) throws IOException {
+ this.tokenHelper = new TokenHelper(new SecurityConfig(conf), secretKeyClient);
+ this.xceiverClientManager = createClientManager(conf, certificateClient);
+ }
+
+ @Nonnull
+ private static XceiverClientManager createClientManager(
+ ConfigurationSource conf, CertificateClient certificateClient)
+ throws IOException {
+ ClientTrustManager trustManager = null;
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
- CACertificateProvider localCaCerts =
- () -> HAUtils.buildCAX509List(certificateClient, conf);
- CACertificateProvider remoteCacerts =
- () -> HAUtils.buildCAX509List(null, conf);
- trustManager = new ClientTrustManager(remoteCacerts, localCaCerts);
++ trustManager = certificateClient.createClientTrustManager();
+ }
+ DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
+ return new XceiverClientManager(conf,
+ new XceiverClientManager.XceiverClientManagerConfigBuilder()
+ .setMaxCacheSize(dnConf.getContainerClientCacheSize())
+ .setStaleThresholdMs(dnConf.getContainerClientCacheStaleThreshold())
+ .build(), trustManager);
+ }
+
+ public XceiverClientManager getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
+ /**
+ * Reads {@link ContainerProtos.ContainerChecksumInfo} for a specified container for the specified datanode.
+ *
+ * @throws IOException For errors communicating with the datanode.
+ * @throws StorageContainerException For errors obtaining the checksum info, including the file being missing or
+ * empty on the datanode, or the datanode not having a replica of the container.
+ * @throws InvalidProtocolBufferException If the file received from the datanode cannot be deserialized.
+ */
+ public ContainerProtos.ContainerChecksumInfo getContainerChecksumInfo(long containerId, DatanodeDetails dn)
+ throws IOException {
+ XceiverClientSpi xceiverClient = this.xceiverClientManager.acquireClient(createSingleNodePipeline(dn));
+ try {
+ String containerToken = encode(tokenHelper.getContainerToken(
+ ContainerID.valueOf(containerId)));
+ ContainerProtos.GetContainerChecksumInfoResponseProto response =
+ ContainerProtocolCalls.getContainerChecksumInfo(xceiverClient,
+ containerId, containerToken);
+ ByteString serializedChecksumInfo = response.getContainerChecksumInfo();
+ // Protobuf will convert an empty ByteString into a default value object. Treat this as an error instead, since
+ // the default value will not represent the state of the container.
+ // The server does not deserialize the file before sending it, so we must check the length on the client.
+ if (serializedChecksumInfo.isEmpty()) {
+ throw new StorageContainerException("Empty Container checksum file for container " + containerId + " received",
+ ContainerProtos.Result.IO_EXCEPTION);
+ } else {
+ return ContainerProtos.ContainerChecksumInfo.parseFrom(serializedChecksumInfo);
+ }
+ } finally {
+ this.xceiverClientManager.releaseClient(xceiverClient, false);
+ }
+ }
+
+ public static Pipeline createSingleNodePipeline(DatanodeDetails dn) {
+ return Pipeline.newBuilder()
+ .setNodes(ImmutableList.of(dn))
+ .setId(PipelineID.valueOf(dn.getUuid()))
+ .setState(Pipeline.PipelineState.CLOSED)
+ .setReplicationConfig(StandaloneReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE)).build();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (xceiverClientManager != null) {
+ xceiverClientManager.close();
+ }
+ }
+}
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ReconcileContainerTask.java
index ac42efd45a,0000000000..5d949e90b1
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ReconcileContainerTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ReconcileContainerTask.java
@@@ -1,88 -1,0 +1,98 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.checksum;
+
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask;
+import org.apache.hadoop.ozone.protocol.commands.ReconcileContainerCommand;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+/**
+ * Used to execute a container reconciliation task that has been queued from the ReplicationSupervisor.
+ */
+public class ReconcileContainerTask extends AbstractReplicationTask {
+ private final ReconcileContainerCommand command;
+ private final DNContainerOperationClient dnClient;
+ private final ContainerController controller;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReconcileContainerTask.class);
+
+ public ReconcileContainerTask(ContainerController controller,
+ DNContainerOperationClient dnClient, ReconcileContainerCommand command) {
+ super(command.getContainerID(), command.getDeadline(), command.getTerm());
+ this.command = command;
+ this.controller = controller;
+ this.dnClient = dnClient;
+ }
+
+ @Override
+ public void runTask() {
+ long start = Time.monotonicNow();
+
+ LOG.info("{}", this);
+
+ try {
+ controller.reconcileContainer(dnClient, command.getContainerID(), command.getPeerDatanodes());
+ setStatus(Status.DONE);
+ long elapsed = Time.monotonicNow() - start;
+ LOG.info("{} completed in {} ms", this, elapsed);
+ } catch (Exception e) {
+ long elapsed = Time.monotonicNow() - start;
+ setStatus(Status.FAILED);
+ LOG.warn("{} failed in {} ms", this, elapsed, e);
+ }
+ }
+
+ @Override
+ protected Object getCommandForDebug() {
+ return command.toString();
+ }
+
++ @Override
++ protected String getMetricName() {
++ return "ContainerReconciliations";
++ }
++
++ @Override
++ protected String getMetricDescriptionSegment() {
++ return "Container Reconciliations";
++ }
++
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReconcileContainerTask that = (ReconcileContainerTask) o;
+ return Objects.equals(command, that.command);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getContainerId());
+ }
+}
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 77a4d97878,f1e637085e..fb9dc49071
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@@ -21,10 -21,9 +21,11 @@@ package org.apache.hadoop.ozone.contain
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Set;
+ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index d102872764,06a4543bd7..06987f6356
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@@ -100,9 -93,7 +100,11 @@@ import org.apache.hadoop.ozone.containe
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
++
+ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_ALREADY_EXISTS;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
@@@ -132,11 -122,8 +135,10 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion;
- import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto.State.RECOVERING;
++
+
+import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
- import static org.apache.hadoop.ozone.ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST;
+ import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST;
-import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
import org.apache.hadoop.util.Time;
import org.apache.ratis.statemachine.StateMachine;
@@@ -677,56 -665,6 +714,47 @@@ public class KeyValueHandler extends Ha
return getEchoResponse(request);
}
+ ContainerCommandResponseProto handleGetContainerChecksumInfo(
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+
+ if (!request.hasGetContainerChecksumInfo()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Malformed Read Container Merkle tree request. trace ID: {}",
+ request.getTraceID());
+ }
+ return malformedRequest(request);
+ }
+
+ // A container must have moved past the CLOSING
+ KeyValueContainerData containerData = kvContainer.getContainerData();
+ State state = containerData.getState();
+ boolean stateSupportsChecksumInfo = (state == CLOSED || state == QUASI_CLOSED || state == UNHEALTHY);
+ if (!stateSupportsChecksumInfo) {
+ return ContainerCommandResponseProto.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setTraceID(request.getTraceID())
+ .setResult(UNCLOSED_CONTAINER_IO)
+ .setMessage("Checksum information is not available for containers in state " + state)
+ .build();
+ }
+
+ ByteString checksumTree = null;
+ try {
+ checksumTree = checksumManager.getContainerChecksumInfo(containerData);
+ } catch (IOException ex) {
+ // For file not found or other inability to read the file, return an error to the client.
+ LOG.error("Error occurred when reading checksum file for container {}", containerData.getContainerID(), ex);
+ return ContainerCommandResponseProto.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setTraceID(request.getTraceID())
+ .setResult(IO_EXCEPTION)
+ .setMessage(ex.getMessage())
+ .build();
+ }
+
+ return getGetContainerMerkleTreeResponse(request, checksumTree);
+ }
+
- /**
- * Checks if a replicaIndex needs to be checked based on the client version for a request.
- * @param request ContainerCommandRequest object.
- * @return true if the validation is required for the client version else false.
- */
- private boolean replicaIndexCheckRequired(ContainerCommandRequestProto request) {
- return request.hasVersion() && request.getVersion() >= EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST.toProtoValue();
- }
-
/**
* Handle Get Block operation. Calls BlockManager to process the request.
*/
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
index 1a4f0bf646,8ff2e30876..af810c6284
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
@@@ -62,8 -63,8 +62,9 @@@ public class BackgroundContainerDataSca
throttler = new HddsDataTransferThrottler(conf.getBandwidthPerVolume());
canceler = new Canceler();
this.metrics = ContainerDataScannerMetrics.create(volume.toString());
+ this.metrics.setStorageDirectory(volume.toString());
this.minScanGap = conf.getContainerScanMinGap();
+ this.checksumManager = checksumManager;
}
private boolean shouldScan(Container<?> container) {
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index cb7db07c24,5307f393e0..c6eaeebfd4
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@@ -34,7 -35,9 +35,10 @@@ import org.apache.hadoop.hdds.security.
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+ import org.apache.hadoop.hdds.utils.db.Table;
+ import org.apache.hadoop.hdds.utils.db.TableIterator;
+ import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
@@@ -517,11 -541,20 +546,21 @@@ public class OzoneContainer
}
blockDeletingService.shutdown();
recoveringContainerScrubbingService.shutdown();
+ IOUtils.closeQuietly(metrics);
ContainerMetrics.remove();
+ checksumTreeManager.stop();
+ if (this.witnessedContainerMetadataStore != null) {
+ try {
+ this.witnessedContainerMetadataStore.stop();
+ } catch (Exception e) {
+ LOG.error("Error while stopping witnessedContainerMetadataStore. Status of store: {}",
+ witnessedContainerMetadataStore.isClosed(), e);
+ }
+ this.witnessedContainerMetadataStore = null;
+ }
}
- public void handleVolumeFailures() {
+ public void handleVolumeFailures() throws StorageContainerException {
if (containerSet != null) {
containerSet.handleVolumeFailures(context);
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 30a8a9bcbc,655ecbb48b..0ff2aaa22b
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@@ -83,7 -68,9 +83,8 @@@ import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
+ import org.mockito.Mockito;
-import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
index 9f93c56f2d,a38b98c53a..89522ded68
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
@@@ -43,11 -44,10 +44,11 @@@ import picocli.CommandLine.Spec
CreateSubcommand.class,
CloseSubcommand.class,
ReportSubcommand.class,
- UpgradeSubcommand.class
+ UpgradeSubcommand.class,
+ ReconcileSubcommand.class
})
- @MetaInfServices(SubcommandWithParent.class)
- public class ContainerCommands implements Callable<Void>, SubcommandWithParent {
+ @MetaInfServices(AdminSubcommand.class)
+ public class ContainerCommands implements Callable<Void>, AdminSubcommand {
@Spec
private CommandSpec spec;
diff --cc hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
index fae0899178,83c0731ff7..e618ebd063
--- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
@@@ -101,6 -84,35 +101,18 @@@ Report containers as JSO
Should contain ${output} stats
Should contain ${output} samples
+ List all containers
+ ${output} = Execute ozone admin container list --all
+ Should contain ${output} OPEN
+
+ List all containers according to count (batchSize)
+ ${output} = Execute ozone admin container list --all --count 10
+ Should contain ${output} OPEN
+
+ List all containers from a particular container ID
+ ${output} = Execute ozone admin container list --all --start 1
+ Should contain ${output} OPEN
+
-Close container
- ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1
- Execute ozone admin container close "${container}"
- ${output} = Execute ozone admin container info "${container}"
- Should contain ${output} CLOS
- Wait until keyword succeeds 1min 10sec Container is closed ${container}
-
-Incomplete command
- ${output} = Execute And Ignore Error ozone admin container
- Should contain ${output} Incomplete command
- Should contain ${output} list
- Should contain ${output} info
- Should contain ${output} create
- Should contain ${output} close
- Should contain ${output} report
- Should contain ${output} upgrade
-
#List containers on unknown host
# ${output} = Execute And Ignore Error ozone admin --verbose container list --scm unknown-host
# Should contain ${output} Invalid host name
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
index c5147ecfb0,49f2d43d63..bd1fbe7382
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
@@@ -17,15 -17,574 +17,575 @@@
package org.apache.hadoop.ozone.client.rpc;
+ import org.apache.commons.lang3.NotImplementedException;
+ import org.apache.hadoop.hdds.HddsConfigKeys;
+ import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+ import org.apache.hadoop.hdds.client.ECReplicationConfig;
+ import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+ import org.apache.hadoop.hdds.client.ReplicationConfig;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.conf.StorageUnit;
+ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+ import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+ import org.apache.hadoop.hdds.scm.container.ContainerID;
+ import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+ import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+ import org.apache.hadoop.hdds.utils.IOUtils;
+ import org.apache.hadoop.ozone.ClientConfigForTesting;
+ import org.apache.hadoop.ozone.MiniOzoneCluster;
+ import org.apache.hadoop.ozone.OzoneConfigKeys;
+ import org.apache.hadoop.ozone.client.BucketArgs;
+ import org.apache.hadoop.ozone.client.ObjectStore;
+ import org.apache.hadoop.ozone.client.OzoneBucket;
+ import org.apache.hadoop.ozone.client.OzoneClient;
+ import org.apache.hadoop.ozone.client.OzoneClientFactory;
+ import org.apache.hadoop.ozone.client.OzoneKey;
+ import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+ import org.apache.hadoop.ozone.client.OzoneVolume;
+ import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
+ import org.apache.hadoop.ozone.client.io.KeyOutputStream;
+ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+ import org.apache.hadoop.ozone.container.TestHelper;
+ import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+ import org.apache.ozone.test.GenericTestUtils;
+ import org.apache.ozone.test.tag.Unhealthy;
+ import org.junit.jupiter.api.AfterAll;
+ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
+ import org.junit.jupiter.api.Test;
+ import org.mockito.MockedStatic;
+ import org.mockito.Mockito;
+
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+ import java.util.UUID;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.TimeoutException;
+ import java.util.concurrent.atomic.AtomicReference;
+
+ import static java.nio.charset.StandardCharsets.UTF_8;
+ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
+ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+ import static org.assertj.core.api.Assertions.assertThat;
+ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+ import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+ import static org.junit.jupiter.api.Assertions.assertNotEquals;
+ import static org.junit.jupiter.api.Assertions.assertThrows;
+ import static org.junit.jupiter.api.Assertions.fail;
+ import static org.mockito.ArgumentMatchers.any;
/**
- * Tests key output stream without zero-copy enabled.
+ * Tests key output stream.
*/
- public class TestECKeyOutputStream extends
- AbstractTestECKeyOutputStream {
+ public class TestECKeyOutputStream {
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ private static OzoneClient client;
+ private static ObjectStore objectStore;
+ private static int chunkSize;
+ private static int flushSize;
+ private static int maxFlushSize;
+ private static int blockSize;
+ private static String volumeName;
+ private static String bucketName;
+ private static String keyString;
+ private static int dataBlocks = 3;
+ private static int inputSize = dataBlocks * chunkSize;
+ private static byte[][] inputChunks = new byte[dataBlocks][chunkSize];
+
+ private static void initConf(OzoneConfiguration configuration) {
+ OzoneClientConfig clientConfig = configuration.getObject(OzoneClientConfig.class);
+ clientConfig.setChecksumType(ContainerProtos.ChecksumType.NONE);
+ clientConfig.setStreamBufferFlushDelay(false);
+ configuration.setFromObject(clientConfig);
+
+ // If SCM detects dead node too quickly, then container would be moved to
+ // closed state and all in progress writes will get exception. To avoid
+ // that, we are just keeping higher timeout and none of the tests depending
+ // on deadnode detection timeout currently.
+ configuration.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS);
+ configuration.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 60, TimeUnit.SECONDS);
+ configuration.setTimeDuration("hdds.ratis.raft.server.rpc.slowness.timeout", 300,
+ TimeUnit.SECONDS);
+ configuration.set("ozone.replication.allowed-configs", "(^((STANDALONE|RATIS)/(ONE|THREE))|(EC/(3-2|6-3|10-4)-" +
+ "(512|1024|2048|4096|1)k)$)");
+ configuration.setTimeDuration(
+ "hdds.ratis.raft.server.notification.no-leader.timeout", 300,
+ TimeUnit.SECONDS);
+ configuration.setQuietMode(false);
+ configuration.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
+ configuration.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 500,
+ TimeUnit.MILLISECONDS);
+ configuration.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1,
+ TimeUnit.SECONDS);
+ configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
+ // "Enable" hsync to verify that hsync would be blocked by ECKeyOutputStream
+ configuration.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ configuration.setBoolean("ozone.client.hbase.enhancements.allowed", true);
+ configuration.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
+
+ ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
+ .setBlockSize(blockSize)
+ .setChunkSize(chunkSize)
+ .setStreamBufferFlushSize(flushSize)
+ .setStreamBufferMaxSize(maxFlushSize)
+ .applyTo(configuration);
+ }
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ */
@BeforeAll
- public static void init() throws Exception {
- init(false);
+ protected static void init() throws Exception {
+ chunkSize = 1024 * 1024;
+ flushSize = 2 * chunkSize;
+ maxFlushSize = 2 * flushSize;
+ blockSize = 2 * maxFlushSize;
+ initConf(conf);
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(10)
+ .build();
+ cluster.waitForClusterToBeReady();
+ client = OzoneClientFactory.getRpcClient(conf);
+ objectStore = client.getObjectStore();
+ keyString = UUID.randomUUID().toString();
+ volumeName = "testeckeyoutputstream";
+ bucketName = volumeName;
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+ initInputChunks();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterAll
+ public static void shutdown() {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testCreateKeyWithECReplicationConfig() throws Exception {
+ try (OzoneOutputStream key = TestHelper
+ .createKey(keyString, new ECReplicationConfig(3, 2,
+ ECReplicationConfig.EcCodec.RS, chunkSize), inputSize,
+ objectStore, volumeName, bucketName)) {
+ assertInstanceOf(ECKeyOutputStream.class, key.getOutputStream());
+ }
+ }
+
+ @Test
+ @Unhealthy("HDDS-11821")
+ public void testECKeyCreatetWithDatanodeIdChange()
+ throws Exception {
+ AtomicReference<Boolean> failed = new AtomicReference<>(false);
+ AtomicReference<MiniOzoneCluster> miniOzoneCluster = new AtomicReference<>();
+ OzoneClient client1 = null;
+ try (MockedStatic<Handler> mockedHandler = Mockito.mockStatic(Handler.class, Mockito.CALLS_REAL_METHODS)) {
+ Map<String, Handler> handlers = new HashMap<>();
- mockedHandler.when(() -> Handler.getHandlerForContainerType(any(), any(), any(), any(), any(), any(), any()))
++ mockedHandler.when(() -> Handler
++ .getHandlerForContainerType(any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenAnswer(i -> {
+ Handler handler = Mockito.spy((Handler) i.callRealMethod());
+ handlers.put(handler.getDatanodeId(), handler);
+ return handler;
+ });
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ initConf(ozoneConfiguration);
+ miniOzoneCluster.set(MiniOzoneCluster.newBuilder(ozoneConfiguration).setNumDatanodes(10).build());
+ miniOzoneCluster.get().waitForClusterToBeReady();
+ client1 = miniOzoneCluster.get().newClient();
+ ObjectStore store = client1.getObjectStore();
+ store.createVolume(volumeName);
+ store.getVolume(volumeName).createBucket(bucketName);
+ OzoneOutputStream key = TestHelper.createKey(keyString, new ECReplicationConfig(3, 2,
+ ECReplicationConfig.EcCodec.RS, 1024), inputSize, store, volumeName, bucketName);
+ byte[] b = new byte[6 * 1024];
+ ECKeyOutputStream groupOutputStream = (ECKeyOutputStream) key.getOutputStream();
+ List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList();
+ while (locationInfoList.isEmpty()) {
+ locationInfoList = groupOutputStream.getLocationInfoList();
+ Random random = new Random();
+ random.nextBytes(b);
+ assertInstanceOf(ECKeyOutputStream.class, key.getOutputStream());
+ key.write(b);
+ key.flush();
+ }
+
+ assertEquals(1, locationInfoList.size());
+
+ OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
+ long containerId = omKeyLocationInfo.getContainerID();
+ Pipeline pipeline = omKeyLocationInfo.getPipeline();
+ DatanodeDetails dnWithReplicaIndex1 =
+ pipeline.getReplicaIndexes().entrySet().stream().filter(e -> e.getValue() == 1).map(Map.Entry::getKey)
+ .findFirst().get();
+ Mockito.when(handlers.get(dnWithReplicaIndex1.getUuidString()).getDatanodeId())
+ .thenAnswer(i -> {
+ if (!failed.get()) {
+ // Change dnId for one write chunk request.
+ failed.set(true);
+ return dnWithReplicaIndex1.getUuidString() + "_failed";
+ } else {
+ return dnWithReplicaIndex1.getUuidString();
+ }
+ });
+ locationInfoList = groupOutputStream.getLocationInfoList();
+ while (locationInfoList.size() == 1) {
+ locationInfoList = groupOutputStream.getLocationInfoList();
+ Random random = new Random();
+ random.nextBytes(b);
+ assertInstanceOf(ECKeyOutputStream.class, key.getOutputStream());
+ key.write(b);
+ key.flush();
+ }
+ assertEquals(2, locationInfoList.size());
+ assertNotEquals(locationInfoList.get(1).getPipeline().getId(), pipeline.getId());
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return miniOzoneCluster.get().getStorageContainerManager().getContainerManager()
+ .getContainer(ContainerID.valueOf(containerId)).getState().equals(
+ HddsProtos.LifeCycleState.CLOSED);
+ } catch (ContainerNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }, 1000, 30000);
+ key.close();
+ Assertions.assertTrue(failed.get());
+ } finally {
+ IOUtils.closeQuietly(client1);
+ if (miniOzoneCluster.get() != null) {
+ miniOzoneCluster.get().shutdown();
+ }
+ }
}
+
+ @Test
+ public void testCreateKeyWithOutBucketDefaults() throws Exception {
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ try (OzoneOutputStream out = bucket.createKey("myKey", inputSize)) {
+ assertInstanceOf(KeyOutputStream.class, out.getOutputStream());
+ for (byte[] inputChunk : inputChunks) {
+ out.write(inputChunk);
+ }
+ }
+ }
+
+ @Test
+ public void testCreateKeyWithBucketDefaults() throws Exception {
+ String myBucket = UUID.randomUUID().toString();
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
+ bucketArgs.setDefaultReplicationConfig(
+ new DefaultReplicationConfig(
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize)));
+
+ volume.createBucket(myBucket, bucketArgs.build());
+ OzoneBucket bucket = volume.getBucket(myBucket);
+
+ try (OzoneOutputStream out = bucket.createKey(keyString, inputSize)) {
+ assertInstanceOf(ECKeyOutputStream.class, out.getOutputStream());
+ for (byte[] inputChunk : inputChunks) {
+ out.write(inputChunk);
+ }
+ }
+ byte[] buf = new byte[chunkSize];
+ try (OzoneInputStream in = bucket.readKey(keyString)) {
+ for (byte[] inputChunk : inputChunks) {
+ int read = in.read(buf, 0, chunkSize);
+ assertEquals(chunkSize, read);
+ assertArrayEquals(buf, inputChunk);
+ }
+ }
+ }
+
+ @Test
+ public void testOverwriteECKeyWithRatisKey() throws Exception {
+ String myBucket = UUID.randomUUID().toString();
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
+ volume.createBucket(myBucket, bucketArgs.build());
+ OzoneBucket bucket = volume.getBucket(myBucket);
+ createKeyAndCheckReplicationConfig(keyString, bucket,
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize));
+
+ //Overwrite with RATIS/THREE
+ createKeyAndCheckReplicationConfig(keyString, bucket,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
+
+ //Overwrite with RATIS/ONE
+ createKeyAndCheckReplicationConfig(keyString, bucket,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE));
+ }
+
+ @Test
+ public void testOverwriteRatisKeyWithECKey() throws Exception {
+ String myBucket = UUID.randomUUID().toString();
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
+ volume.createBucket(myBucket, bucketArgs.build());
+ OzoneBucket bucket = volume.getBucket(myBucket);
+
+ createKeyAndCheckReplicationConfig(keyString, bucket,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
+ // Overwrite with EC key
+ createKeyAndCheckReplicationConfig(keyString, bucket,
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize));
+ }
+
+ private void createKeyAndCheckReplicationConfig(String keyName,
+ OzoneBucket bucket, ReplicationConfig replicationConfig)
+ throws IOException {
+ try (OzoneOutputStream out = bucket
+ .createKey(keyName, inputSize, replicationConfig, new HashMap<>())) {
+ for (byte[] inputChunk : inputChunks) {
+ out.write(inputChunk);
+ }
+ }
+ OzoneKeyDetails key = bucket.getKey(keyName);
+ assertEquals(replicationConfig, key.getReplicationConfig());
+ }
+
+ @Test
+ public void testCreateRatisKeyAndWithECBucketDefaults() throws Exception {
+ OzoneBucket bucket = getOzoneBucket();
+ try (OzoneOutputStream out = bucket.createKey(
+ "testCreateRatisKeyAndWithECBucketDefaults", 2000,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
+ new HashMap<>())) {
+ assertInstanceOf(KeyOutputStream.class, out.getOutputStream());
+ for (byte[] inputChunk : inputChunks) {
+ out.write(inputChunk);
+ }
+ }
+ }
+
+ @Test
+ public void test13ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(13);
+ }
+
+ @Test
+ public void testChunksInSingleWriteOpWithOffset() throws IOException {
+ testMultipleChunksInSingleWriteOp(11, 25, 19);
+ }
+
+ @Test
+ public void test15ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(15);
+ }
+
+ @Test
+ public void test20ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(20);
+ }
+
+ @Test
+ public void test21ChunksInSingleWriteOp() throws IOException {
+ testMultipleChunksInSingleWriteOp(21);
+ }
+
+ private void testMultipleChunksInSingleWriteOp(int offset,
+ int bufferChunks, int numChunks)
+ throws IOException {
+ byte[] inputData = getInputBytes(offset, bufferChunks, numChunks);
+ final OzoneBucket bucket = getOzoneBucket();
+ String keyName =
+ String.format("testMultipleChunksInSingleWriteOpOffset" +
+ "%dBufferChunks%dNumChunks", offset, bufferChunks,
+ numChunks);
+ try (OzoneOutputStream out = bucket.createKey(keyName, 4096,
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize), new HashMap<>())) {
+ out.write(inputData, offset, numChunks * chunkSize);
+ }
+
+ validateContent(offset, numChunks * chunkSize, inputData, bucket,
+ bucket.getKey(keyName));
+ }
+
+ private void testMultipleChunksInSingleWriteOp(int numChunks)
+ throws IOException {
+ testMultipleChunksInSingleWriteOp(0, numChunks, numChunks);
+ }
+
+ @Test
+ public void testECContainerKeysCountAndNumContainerReplicas()
+ throws IOException, InterruptedException, TimeoutException {
+ byte[] inputData = getInputBytes(1);
+ final OzoneBucket bucket = getOzoneBucket();
+ ContainerOperationClient containerOperationClient =
+ new ContainerOperationClient(conf);
+
+ ECReplicationConfig repConfig = new ECReplicationConfig(
+ 3, 2, ECReplicationConfig.EcCodec.RS, chunkSize);
+ // Close all EC pipelines so we must get a fresh pipeline and hence
+ // container for this test.
+ PipelineManager pm =
+ cluster.getStorageContainerManager().getPipelineManager();
+ for (Pipeline p : pm.getPipelines(repConfig)) {
+ pm.closePipeline(p, true);
+ }
+
+ String keyName = UUID.randomUUID().toString();
+ try (OzoneOutputStream out = bucket.createKey(keyName, 4096,
+ repConfig, new HashMap<>())) {
+ out.write(inputData);
+ }
+ OzoneKeyDetails key = bucket.getKey(keyName);
+ long currentKeyContainerID =
+ key.getOzoneKeyLocations().get(0).getContainerID();
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return (containerOperationClient.getContainer(currentKeyContainerID)
+ .getNumberOfKeys() == 1) && (containerOperationClient
+ .getContainerReplicas(currentKeyContainerID).size() == 5);
+ } catch (IOException exception) {
+ fail("Unexpected exception " + exception);
+ return false;
+ }
+ }, 100, 10000);
+ validateContent(inputData, bucket, key);
+ }
+
+ private void validateContent(byte[] inputData, OzoneBucket bucket,
+ OzoneKey key) throws IOException {
+ validateContent(0, inputData.length, inputData, bucket, key);
+ }
+
+ private void validateContent(int offset, int length, byte[] inputData,
+ OzoneBucket bucket,
+ OzoneKey key) throws IOException {
+ try (OzoneInputStream is = bucket.readKey(key.getName())) {
+ byte[] fileContent = new byte[length];
+ assertEquals(length, is.read(fileContent));
+ assertEquals(new String(Arrays.copyOfRange(inputData, offset,
+ offset + length), UTF_8),
+ new String(fileContent, UTF_8));
+ }
+ }
+
+ private OzoneBucket getOzoneBucket() throws IOException {
+ String myBucket = UUID.randomUUID().toString();
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
+ bucketArgs.setDefaultReplicationConfig(
+ new DefaultReplicationConfig(
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize)));
+
+ volume.createBucket(myBucket, bucketArgs.build());
+ return volume.getBucket(myBucket);
+ }
+
+ private static void initInputChunks() {
+ for (int i = 0; i < dataBlocks; i++) {
+ inputChunks[i] = getBytesWith(i + 1, chunkSize);
+ }
+ }
+
+ private static byte[] getBytesWith(int singleDigitNumber, int total) {
+ StringBuilder builder = new StringBuilder(singleDigitNumber);
+ for (int i = 1; i <= total; i++) {
+ builder.append(singleDigitNumber);
+ }
+ return builder.toString().getBytes(UTF_8);
+ }
+
+ @Test
+ public void testWriteShouldSucceedWhenDNKilled() throws Exception {
+ int numChunks = 3;
+ byte[] inputData = getInputBytes(numChunks);
+ final OzoneBucket bucket = getOzoneBucket();
+ String keyName = "testWriteShouldSucceedWhenDNKilled" + numChunks;
+ DatanodeDetails nodeToKill = null;
+ try {
+ try (OzoneOutputStream out = bucket.createKey(keyName, 1024,
+ new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS,
+ chunkSize), new HashMap<>())) {
+ ECKeyOutputStream ecOut = (ECKeyOutputStream) out.getOutputStream();
+ out.write(inputData);
+ // Kill a node from first pipeline
+ nodeToKill = ecOut.getStreamEntries()
+ .get(0).getPipeline().getFirstNode();
+ cluster.shutdownHddsDatanode(nodeToKill);
+
+ out.write(inputData);
+
+ // Wait for flushing thread to finish its work.
+ final long checkpoint = System.currentTimeMillis();
+ ecOut.insertFlushCheckpoint(checkpoint);
+ GenericTestUtils.waitFor(() -> ecOut.getFlushCheckpoint() == checkpoint,
+ 100, 10000);
+
+ // Check the second blockGroup pipeline to make sure that the failed
+ // node is not selected.
+ assertThat(ecOut.getStreamEntries().get(1).getPipeline().getNodes())
+ .doesNotContain(nodeToKill);
+ }
+
+ try (OzoneInputStream is = bucket.readKey(keyName)) {
+ // We wrote "inputData" twice, so do two reads and ensure the correct
+ // data comes back.
+ for (int i = 0; i < 2; i++) {
+ byte[] fileContent = new byte[inputData.length];
+ assertEquals(inputData.length, is.read(fileContent));
+ assertEquals(new String(inputData, UTF_8),
+ new String(fileContent, UTF_8));
+ }
+ }
+ } finally {
+ cluster.restartHddsDatanode(nodeToKill, true);
+ }
+ }
+
+ private byte[] getInputBytes(int numChunks) {
+ return getInputBytes(0, numChunks, numChunks);
+ }
+
+ private byte[] getInputBytes(int offset, int bufferChunks, int numChunks) {
+ byte[] inputData = new byte[offset + bufferChunks * chunkSize];
+ for (int i = 0; i < numChunks; i++) {
+ int start = offset + (i * chunkSize);
+ Arrays.fill(inputData, start, start + chunkSize - 1,
+ String.valueOf(i % 9).getBytes(UTF_8)[0]);
+ }
+ return inputData;
+ }
+
+ @Test
+ public void testBlockedHflushAndHsync() throws Exception {
+ // Expect ECKeyOutputStream hflush and hsync calls to throw exception
+ try (OzoneOutputStream oOut = TestHelper.createKey(
+ keyString, new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, chunkSize),
+ inputSize, objectStore, volumeName, bucketName)) {
+ assertInstanceOf(ECKeyOutputStream.class, oOut.getOutputStream());
+ KeyOutputStream kOut = (KeyOutputStream) oOut.getOutputStream();
+
+ assertThrows(NotImplementedException.class, () -> kOut.hflush());
+ assertThrows(NotImplementedException.class, () -> kOut.hsync());
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org