You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2026/05/05 13:22:40 UTC

(ozone) 01/02: Merge branch 'master' into merge-master

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git

commit 98d066f5cbecc683813e4e890894987078c80e21
Merge: 4ab7a96b0b3 99b53f44436
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Thu Apr 30 19:01:05 2026 +0100

    Merge branch 'master' into merge-master
    
     Conflicts:
            hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
            hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
            hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
            hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java

 .github/ci.md                                      |     7 -
 .github/comment-commands/debug.sh                  |    21 -
 .github/comment-commands/help.sh                   |    27 -
 .github/comment-commands/label.sh                  |    25 -
 .github/comment-commands/retest.sh                 |    75 -
 .github/dependabot.yml                             |    10 +
 .github/process-comment.sh                         |    56 -
 .github/pull_request_template.md                   |     3 +
 .github/workflows/build-ratis.yml                  |    34 +-
 .github/workflows/check.yml                        |    32 +-
 .github/workflows/ci-with-ratis.yml                |     9 +-
 .github/workflows/ci.yml                           |    90 +-
 .github/workflows/close-stale-prs.yaml             |     6 +-
 .github/workflows/generate-config-doc.yml          |    61 +
 .github/workflows/intermittent-test-check.yml      |    68 +-
 .github/workflows/label-pr.yml                     |    36 +-
 .github/workflows/populate-cache.yml               |    14 +-
 .github/workflows/post-commit.yml                  |     7 +-
 .github/workflows/pull-request.yml                 |     8 +-
 .github/workflows/repeat-acceptance.yml            |    24 +-
 .../schedule-label-pr.yml}                         |    18 +-
 .github/workflows/scheduled-cache-update.yml       |     3 +-
 .github/workflows/update-ozone-site-config-doc.yml |   186 +
 .github/workflows/{comments.yaml => zizmor.yml}    |    35 +-
 CONTRIBUTING.md                                    |     5 +-
 README.md                                          |    21 +-
 SECURITY.md                                        |     2 +-
 dev-support/ci/pr_body_config_doc.sh               |    54 +
 dev-support/ci/selective_ci_checks.sh              |     6 +
 dev-support/ci/xml_to_md.py                        |   177 +
 dev-support/pmd/pmd-ruleset.xml                    |     5 +
 hadoop-hdds/cli-common/pom.xml                     |    16 +-
 .../org/apache/hadoop/hdds/cli/ItemsFromStdin.java |    20 +-
 hadoop-hdds/client/pom.xml                         |     1 -
 .../apache/hadoop/hdds/scm/OzoneClientConfig.java  |    33 +-
 .../apache/hadoop/hdds/scm/XceiverClientGrpc.java  |   186 +-
 .../hadoop/hdds/scm/client/HddsClientUtils.java    |    42 +-
 .../hdds/scm/storage/BlockDataStreamOutput.java    |     8 +
 .../hdds/scm/storage/StreamBlockInputStream.java   |    48 +-
 .../hadoop/ozone/client/io/ECBlockInputStream.java |     6 +
 .../scm/storage/TestStreamBlockInputStream.java    |   193 +-
 .../ozone/client/io/TestECBlockInputStream.java    |    53 +
 .../io/TestECBlockReconstructedInputStream.java    |    10 +-
 .../TestECBlockReconstructedStripeInputStream.java |     9 +-
 hadoop-hdds/common/pom.xml                         |    22 +-
 .../main/java/com/google/protobuf/ProtoUtils.java  |    32 -
 .../org/apache/hadoop/hdds/HddsConfigKeys.java     |     3 +
 .../org/apache/hadoop/hdds/client/BlockID.java     |     4 +-
 .../hadoop/hdds/conf/OzoneConfiguration.java       |     2 +
 .../hadoop/hdds/protocol/DatanodeDetails.java      |    11 +
 .../java/org/apache/hadoop/hdds/scm/ScmConfig.java |    10 +-
 .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java  |    10 +-
 .../hadoop/hdds/scm/container/ContainerInfo.java   |    37 +
 .../balancer/ContainerBalancerConfiguration.java   |    68 +-
 .../org/apache/hadoop/hdds/scm/net/InnerNode.java  |     5 -
 .../java/org/apache/hadoop/hdds/scm/net/Node.java  |    13 -
 .../apache/hadoop/hdds/scm/pipeline/Pipeline.java  |    25 +-
 .../hadoop/hdds/security/SecurityConfig.java       |    67 +
 .../apache/hadoop/hdds/tracing/LoopSampler.java}   |    28 +-
 .../apache/hadoop/hdds/tracing/SkipTracing.java    |    18 +-
 .../apache/hadoop/hdds/tracing/SpanSampler.java    |    90 +
 .../apache/hadoop/hdds/tracing/TraceAllMethod.java |    31 +-
 .../apache/hadoop/hdds/tracing/TracingConfig.java  |   128 +
 .../apache/hadoop/hdds/tracing/TracingUtil.java    |   160 +-
 .../apache/hadoop/hdds/utils/SimpleStriped.java    |    39 +-
 .../apache/hadoop/hdds/utils/SlidingWindow.java    |    11 +-
 .../apache/hadoop/hdds/utils/db/IntegerCodec.java  |    14 +-
 .../org/apache/hadoop/hdds/utils/db/LongCodec.java |    21 +-
 .../hdds/utils/io/RandomAccessFileChannel.java     |    37 +-
 .../org/apache/hadoop/ipc_/AlignmentContext.java   |     2 +-
 .../main/java/org/apache/hadoop/ipc_/Server.java   |     2 +-
 .../org/apache/hadoop/ozone/OzoneConfigKeys.java   |    31 +-
 .../java/org/apache/hadoop/ozone/OzoneConsts.java  |     2 +
 .../apache/hadoop/ozone/OzoneManagerVersion.java   |     5 +-
 .../org/apache/hadoop/ozone/OzoneSecurityUtil.java |    21 +
 .../org/apache/hadoop/ozone/common/Checksum.java   |     4 +-
 .../ozone/common/PureJavaCrc32ByteBuffer.java      |    12 +
 .../ozone/common/PureJavaCrc32CByteBuffer.java     |    12 +
 .../ozone/container/common/helpers/BlockData.java  |     4 +-
 .../common/src/main/resources/ozone-default.xml    |   243 +-
 .../security/TestSecurityConfigTlsSettings.java    |    90 +
 .../hadoop/hdds/tracing/TestSpanSampling.java      |   144 +
 .../hadoop/hdds/tracing/TestTraceAllMethod.java    |    34 +
 .../hadoop/hdds/tracing/TestTracingUtil.java       |    50 +-
 .../hadoop/hdds/utils/TestSimpleStriped.java       |    14 -
 .../hdds/utils/io/TestRandomAccessFileChannel.java |   153 +
 hadoop-hdds/config/pom.xml                         |    10 +
 .../hadoop/hdds/conf/ConfigFileAppender.java       |     8 +-
 .../hadoop/hdds/conf/ConfigFileGenerator.java      |     2 +
 hadoop-hdds/container-service/pom.xml              |     1 -
 .../apache/hadoop/ozone/HddsDatanodeService.java   |    27 +-
 .../checksum/ContainerChecksumTreeManager.java     |     3 +-
 .../ozone/container/common/impl/ContainerData.java |    37 +-
 .../container/common/impl/ContainerDataYaml.java   |     3 +-
 .../container/common/impl/HddsDispatcher.java      |     7 +-
 .../common/statemachine/DatanodeConfiguration.java |   200 +-
 .../commandhandler/DeleteBlocksCommandHandler.java |     6 +-
 .../states/endpoint/VersionEndpointTask.java       |     1 +
 .../common/transport/server/XceiverServerGrpc.java |     2 +
 .../server/ratis/ContainerStateMachine.java        |    22 +-
 .../ozone/container/common/volume/HddsVolume.java  |    44 +-
 .../container/common/volume/MutableVolumeSet.java  |    20 +-
 .../container/common/volume/StorageVolume.java     |   154 +-
 .../common/volume/StorageVolumeChecker.java        |   114 +-
 .../common/volume/ThrottledAsyncChecker.java       |     2 +-
 .../common/volume/VolumeChoosingPolicyFactory.java |    11 +-
 .../ContainerChoosingPolicyFactory.java            |    55 +
 .../diskbalancer/DiskBalancerConfiguration.java    |   156 +-
 .../container/diskbalancer/DiskBalancerInfo.java   |    55 +-
 .../diskbalancer/DiskBalancerProtocolServer.java   |     5 +-
 .../diskbalancer/DiskBalancerService.java          |   168 +-
 .../container/diskbalancer/DiskBalancerYaml.java   |    20 +-
 ...ChoosingPolicy.java => ContainerCandidate.java} |    42 +-
 .../policy/ContainerChoosingPolicy.java            |    36 +-
 .../policy/DefaultContainerChoosingPolicy.java     |   208 +-
 .../policy/DefaultVolumeChoosingPolicy.java        |   156 -
 .../ECReconstructionCoordinator.java               |    10 -
 .../ozone/container/keyvalue/KeyValueHandler.java  |    16 +-
 .../keyvalue/impl/FilePerBlockStrategy.java        |    27 +-
 .../ozone/container/metadata/AbstractRDBStore.java |    14 +-
 .../metadata/DatanodeSchemaThreeDBDefinition.java  |     6 +-
 .../ozone/container/ozoneimpl/ContainerReader.java |     4 +-
 .../container/replication/ReplicationServer.java   |     2 +
 .../ScmHAFinalizeUpgradeActionDatanode.java        |     1 +
 .../commands/ReplicateContainerCommand.java        |    10 +-
 .../ozone/container/common/SCMTestUtils.java       |    36 +
 .../container/common/TestDatanodeStateMachine.java |    11 +-
 .../container/common/impl/TestHddsDispatcher.java  |    43 +
 .../TestReconstructECContainersCommandHandler.java |     4 +-
 .../states/endpoint/TestHeartbeatEndpointTask.java |     4 +-
 .../server/ratis/TestContainerStateMachine.java    |    43 +-
 .../container/common/volume/TestHddsVolume.java    |    39 +
 .../common/volume/TestStorageVolumeChecker.java    |   185 +
 .../volume/TestStorageVolumeHealthChecks.java      |   149 +-
 .../TestDefaultContainerChoosingPolicy.java        |   780 +-
 .../TestDefaultVolumeChoosingPolicy.java           |   651 --
 .../TestDiskBalancerProtocolServer.java            |    65 +-
 .../diskbalancer/TestDiskBalancerService.java      |    81 +-
 .../diskbalancer/TestDiskBalancerTask.java         |    26 +-
 .../diskbalancer/TestDiskBalancerYaml.java         |    79 +-
 .../keyvalue/impl/TestFilePerBlockStrategy.java    |    51 +
 .../TestBackgroundContainerDataScanner.java        |   114 +
 .../replication/TestReplicationSupervisor.java     |     4 +-
 .../TestDatanodeUpgradeToContainerIdsTable.java    |    12 +-
 .../upgrade/TestDatanodeUpgradeToHBaseSupport.java |    14 +-
 .../upgrade/TestDatanodeUpgradeToSchemaV3.java     |    53 +-
 .../TestReconstructionECContainersCommands.java    |     6 +-
 hadoop-hdds/docs/content/design/diskbalancer.md    |    28 +-
 .../docs/content/design/mpu-gc-optimization.md     |   668 ++
 .../docs/content/design/s3-conditional-requests.md |   668 +-
 .../content/design/storage-distribution-design.jpg |   Bin 0 -> 69109 bytes
 .../docs/content/design/storage-distribution.md    |   427 +
 hadoop-hdds/docs/content/feature/DiskBalancer.md   |    42 +-
 .../docs/content/feature/DiskBalancer.zh.md        |     6 +-
 .../docs/content/feature/Reconfigurability.md      |    28 +-
 .../feature/Snapshot-Configuration-Properties.md   |     1 +
 hadoop-hdds/docs/content/feature/Snapshot.md       |     9 +-
 .../content/feature/SnapshotDefragmentation.md     |   209 +-
 .../apache/ozone/erasurecode/TestCoderBase.java    |    23 +-
 .../erasurecode/rawcoder/TestRawCoderBase.java     |     2 +-
 hadoop-hdds/framework/pom.xml                      |    19 +-
 .../hadoop/hdds/conf/ReconfigurationHandler.java   |    24 +-
 .../hdds/conf/TracingReconfigurationCallback.java  |    58 +
 .../hadoop/hdds/fs/CachingSpaceUsageSource.java    |    12 +-
 .../apache/hadoop/hdds/scm/client/ScmClient.java   |    33 +-
 .../hadoop/hdds/scm/client/ScmTopologyClient.java  |     2 +-
 .../org/apache/hadoop/hdds/scm/ha/SCMHandler.java  |    18 +-
 .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java  |     2 +-
 .../hadoop/hdds/scm/net/NetworkTopologyImpl.java   |    16 +-
 .../hadoop/hdds/scm/net/NodeSchemaLoader.java      |     5 +-
 .../hadoop/hdds/scm/net/NodeSchemaManager.java     |    12 +
 .../apache/hadoop/hdds/scm/net}/package-info.java  |     4 +-
 .../protocol/StorageContainerLocationProtocol.java |    49 +-
 ...inerLocationProtocolClientSideTranslatorPB.java |   137 +-
 .../scm/proxy/SCMFailoverProxyProviderBase.java    |    58 +-
 .../symmetric/DefaultSecretKeySignerClient.java    |     2 +-
 .../hdds/security/symmetric/SecretKeyState.java    |     9 +-
 .../certificate/authority/CertificateStore.java    |     8 +-
 .../client/DefaultCertificateClient.java           |     6 +-
 .../certificate/utils/SelfSignedCertificate.java   |     2 +-
 .../org/apache/hadoop/hdds/server/OzoneAdmins.java |     2 +
 .../apache/hadoop/hdds/server/OzoneBlacklist.java  |   219 +
 .../org/apache/hadoop/hdds/server/ServerUtils.java |    39 +
 .../org/apache/hadoop/hdds/server/YamlUtils.java   |     0
 .../hadoop/hdds/server/http/BaseHttpServer.java    |    28 +-
 .../hadoop/hdds/server/http/HttpServer2.java       |    35 +-
 .../hadoop/hdds/server/http/ProfileServlet.java    |     4 +-
 .../org/apache/hadoop/hdds/utils/Archiver.java     |    38 +-
 .../hadoop/hdds/utils/DBCheckpointServlet.java     |    14 +-
 .../java/org/apache/hadoop/hdds/utils/HAUtils.java |    48 +-
 .../apache/hadoop/hdds/utils/HttpServletUtils.java |     2 +-
 .../utils/db/InodeMetadataRocksDBCheckpoint.java   |    83 +-
 .../hdds/utils/db/RDBStoreAbstractIterator.java    |    19 +-
 .../org/apache/hadoop/hdds/utils/db/RDBTable.java  |     4 +
 .../apache/hadoop/hdds/utils/db/RocksDatabase.java |    23 +-
 .../org/apache/hadoop/ozone/common/BlockGroup.java |     9 +-
 .../apache/hadoop/ozone/common/DeletedBlock.java   |    17 +-
 .../main/resources/network-topology-default.xml    |     0
 .../main/resources/network-topology-default.yaml   |     0
 .../main/resources/network-topology-nodegroup.xml  |     0
 .../src/main/resources/webapps/static/ozone.js     |     2 +
 .../resources/webapps/static/templates/menu.html   |     2 +
 .../hdds/fs/TestCachingSpaceUsageSource.java       |    35 +
 .../hdds/scm/net/TestNetworkTopologyImpl.java      |     0
 .../hadoop/hdds/scm/net/TestNodeSchemaLoader.java  |     2 +-
 .../hadoop/hdds/scm/net/TestNodeSchemaManager.java |     0
 .../hadoop/hdds/scm/net/TestYamlSchemaLoader.java  |     0
 .../hdds/security/ssl/TestGrpcTlsConfig.java       |   253 +
 .../apache/hadoop/hdds/server/TestServerUtils.java |   124 +
 .../hdds/server/http/TestHttpServer2SSL.java       |   305 +
 .../org/apache/hadoop/hdds/utils/TestArchiver.java |    18 +-
 .../hdds/utils/db/TestFixedLengthStringCodec.java  |     7 +-
 .../utils/db/TestRDBStoreIteratorWithDBClose.java  |   308 +
 .../networkTopologyTestFiles/enforce-error.xml     |     0
 .../networkTopologyTestFiles/external-entity.xml   |     0
 .../resources/networkTopologyTestFiles/good.xml    |     0
 .../resources/networkTopologyTestFiles/good.yaml   |     0
 .../networkTopologyTestFiles/invalid-cost.xml      |     0
 .../networkTopologyTestFiles/invalid-version.xml   |     0
 .../networkTopologyTestFiles/middle-leaf.yaml      |     0
 .../networkTopologyTestFiles/multiple-leaf.xml     |     0
 .../networkTopologyTestFiles/multiple-root.xml     |     0
 .../networkTopologyTestFiles/multiple-root.yaml    |     0
 .../networkTopologyTestFiles/multiple-topology.xml |     0
 .../resources/networkTopologyTestFiles/no-leaf.xml |     0
 .../resources/networkTopologyTestFiles/no-root.xml |     0
 .../networkTopologyTestFiles/no-topology.xml       |     0
 .../path-layers-size-mismatch.xml                  |     0
 .../path-with-id-reference-failure.xml             |     0
 .../unknown-layer-type.xml                         |     0
 .../wrong-path-order-1.xml                         |     0
 .../wrong-path-order-2.xml                         |     0
 .../src/main/proto/ScmAdminProtocol.proto          |    42 +-
 .../interface-client/src/main/proto/hdds.proto     |    17 +
 .../proto/ScmServerDatanodeHeartbeatProtocol.proto |     1 +
 .../src/main/proto/ScmServerProtocol.proto         |     1 +
 .../utils/db/managed/ManagedRocksIterator.java     |    31 +-
 hadoop-hdds/rocks-native/pom.xml                   |     1 -
 hadoop-hdds/rocksdb-checkpoint-differ/pom.xml      |     1 -
 hadoop-hdds/server-scm/pom.xml                     |     6 +-
 .../org/apache/hadoop/hdds/scm/FetchMetrics.java   |     2 +-
 .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java |     5 +
 .../scm/block/DeletedBlockLogStateManager.java     |    33 +-
 .../scm/block/DeletedBlockLogStateManagerImpl.java |    50 +-
 .../container/AbstractContainerReportHandler.java  |    31 +-
 .../hdds/scm/container/ContainerManager.java       |    39 +-
 .../hdds/scm/container/ContainerManagerImpl.java   |    40 +-
 .../hdds/scm/container/ContainerStateManager.java  |    36 +-
 .../scm/container/ContainerStateManagerImpl.java   |    43 +-
 .../scm/container/balancer/ContainerBalancer.java  |    49 +-
 .../ContainerBalancerSelectionCriteria.java        |   167 +-
 .../container/balancer/ContainerBalancerTask.java  |     5 +-
 .../hdds/scm/container/balancer/MoveManager.java   |   109 +-
 .../algorithms/SCMContainerPlacementRackAware.java |     2 -
 .../container/placement/metrics/SCMMetrics.java    |    36 +-
 .../replication/ECUnderReplicationHandler.java     |     6 +-
 ...nQueue.java => MonitoringReplicationQueue.java} |     4 +-
 .../QuasiClosedStuckOverReplicationHandler.java    |     4 +-
 .../replication/QuasiClosedStuckReplicaCount.java  |    77 +-
 .../QuasiClosedStuckUnderReplicationHandler.java   |    13 +-
 .../replication/RatisUnderReplicationHandler.java  |     9 +
 .../container/replication/ReplicationManager.java  |    74 +-
 .../replication/health/EmptyContainerHandler.java  |    74 +-
 .../health/QuasiClosedStuckReplicationCheck.java   |    20 +-
 .../scm/container/states/ContainerStateMap.java    |    14 +
 .../hdds/scm/ha/InterSCMGrpcProtocolService.java   |     2 +
 .../apache/hadoop/hdds/scm/ha/ReflectionUtil.java  |    67 -
 .../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java |    10 +-
 .../hadoop/hdds/scm/ha/SCMHAManagerStub.java       |    35 +-
 .../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java |    38 +-
 .../hadoop/hdds/scm/ha/SCMRatisResponse.java       |    20 +-
 .../apache/hadoop/hdds/scm/ha/SCMRatisServer.java  |    10 +-
 .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java     |     7 +-
 .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java |    57 +-
 .../hadoop/hdds/scm/ha/SequenceIdGenerator.java    |    11 +-
 .../apache/hadoop/hdds/scm/ha/StatefulService.java |    48 +-
 .../hdds/scm/ha/StatefulServiceStateManager.java   |     8 +-
 .../scm/ha/StatefulServiceStateManagerImpl.java    |     4 +-
 .../DeletedBlockLogStateManagerInvoker.java        |   148 +
 .../hadoop/hdds/scm/ha/invoker/ScmInvoker.java     |    82 +
 .../scm/ha/invoker/ScmInvokerCodeGenerator.java    |   533 +
 .../hadoop/hdds/scm/ha/invoker}/package-info.java  |     5 +-
 .../apache/hadoop/hdds/scm/ha/io/CodecFactory.java |    72 -
 .../apache/hadoop/hdds/scm/ha/io/EnumCodec.java    |    53 -
 .../apache/hadoop/hdds/scm/ha/io/ListCodec.java    |    81 -
 ...igIntegerCodec.java => ScmBigIntegerCodec.java} |     8 +-
 .../io/{BooleanCodec.java => ScmBooleanCodec.java} |    10 +-
 .../hadoop/hdds/scm/ha/io/ScmByteStringCodec.java  |    11 +-
 .../hdds/scm/ha/io/{Codec.java => ScmCodec.java}   |    12 +-
 .../hadoop/hdds/scm/ha/io/ScmCodecFactory.java     |   178 +
 .../apache/hadoop/hdds/scm/ha/io/ScmEnumCodec.java |    70 +
 .../io/{IntegerCodec.java => ScmIntegerCodec.java} |    15 +-
 .../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java |    81 +
 .../ha/io/{LongCodec.java => ScmLongCodec.java}    |    17 +-
 ...KeyCodec.java => ScmManagedSecretKeyCodec.java} |    10 +-
 ...Codec.java => ScmNonShadedByteStringCodec.java} |    17 +-
 ...java => ScmNonShadedGeneratedMessageCodec.java} |    33 +-
 .../io/{StringCodec.java => ScmStringCodec.java}   |    10 +-
 ...cateCodec.java => ScmX509CertificateCodec.java} |    14 +-
 .../hdds/scm/node/DatanodeAdminMonitorImpl.java    |    10 +-
 .../apache/hadoop/hdds/scm/node/DatanodeInfo.java  |    17 +-
 .../hadoop/hdds/scm/node/DeadNodeHandler.java      |    39 +-
 .../apache/hadoop/hdds/scm/node/NodeManager.java   |    13 +
 .../hadoop/hdds/scm/node/NodeStateManager.java     |     4 +-
 .../hdds/scm/node/PendingContainerTracker.java     |   225 +
 .../hadoop/hdds/scm/node/SCMNodeManager.java       |    30 +
 .../hadoop/hdds/scm/node/SCMNodeMetrics.java       |    24 +
 .../hadoop/hdds/scm/pipeline/PipelineManager.java  |    21 +-
 .../hdds/scm/pipeline/PipelineManagerImpl.java     |    17 +-
 .../hdds/scm/pipeline/PipelinePlacementPolicy.java |     2 -
 .../hdds/scm/pipeline/PipelineStateManager.java    |     9 +-
 .../scm/pipeline/PipelineStateManagerImpl.java     |     4 +-
 ...inerLocationProtocolServerSideTranslatorPB.java |    73 +-
 .../scm/safemode/HealthyPipelineSafeModeRule.java  |    79 +
 .../safemode/OneReplicaPipelineSafeModeRule.java   |    28 +
 .../hadoop/hdds/scm/safemode/SafeModeMetrics.java  |     8 +-
 .../hdds/scm/security/RootCARotationHandler.java   |     9 +-
 .../scm/security/RootCARotationHandlerImpl.java    |     4 +-
 .../hdds/scm/security/RootCARotationManager.java   |    17 +-
 .../scm/security/ScmSecretKeyStateBuilder.java     |     4 +-
 .../hdds/scm/server/SCMBlockProtocolServer.java    |     3 +-
 .../hadoop/hdds/scm/server/SCMCertStore.java       |     4 +-
 .../hdds/scm/server/SCMClientProtocolServer.java   |   125 +-
 .../hdds/scm/server/SCMDBCheckpointServlet.java    |    10 +-
 .../hdds/scm/server/SCMDatanodeProtocolServer.java |    12 +-
 .../hdds/scm/server/SCMHTTPServerConfig.java       |    20 +-
 .../hdds/scm/server/StorageContainerManager.java   |    44 +-
 .../server/StorageContainerManagerHttpServer.java  |     2 +-
 .../scm/server/StorageContainerManagerStarter.java |     2 -
 .../server/upgrade/FinalizationStateManager.java   |     9 +-
 .../upgrade/FinalizationStateManagerImpl.java      |     4 +-
 .../org/apache/hadoop/ozone/audit/SCMAction.java   |     5 +-
 .../main/resources/webapps/scm/ratis-events.html   |    32 +-
 .../main/resources/webapps/scm/scm-overview.html   |     2 +-
 .../src/main/resources/webapps/scm/scm.js          |    26 +
 .../org/apache/hadoop/hdds/scm/HddsTestUtils.java  |     1 +
 .../hadoop/hdds/scm/block/TestDeletedBlockLog.java |    10 +-
 .../hadoop/hdds/scm/container/MockNodeManager.java |    37 +-
 .../hdds/scm/container/SimpleMockNodeManager.java  |    12 +-
 .../scm/container/TestContainerManagerImpl.java    |    19 +-
 .../scm/container/TestContainerReportHandler.java  |   105 +-
 .../scm/container/TestContainerStateManager.java   |    58 +-
 .../hdds/scm/container/balancer/MockedSCM.java     |     7 +
 .../container/balancer/TestContainerBalancer.java  |    37 +
 .../TestContainerBalancerDatanodeNodeLimit.java    |    17 +
 .../TestContainerBalancerSelectionCriteria.java    |   320 +
 .../balancer/TestContainerBalancerTask.java        |    38 +
 .../scm/container/balancer/TestMoveManager.java    |   175 +
 .../algorithms/TestContainerPlacementFactory.java  |     2 +-
 .../TestSCMContainerPlacementCapacity.java         |     3 +-
 .../TestSCMContainerPlacementRackAware.java        |     6 +-
 .../TestSCMContainerPlacementRackScatter.java      |     6 +-
 .../TestSCMContainerPlacementRandom.java           |     6 +-
 .../container/replication/ReplicationTestUtil.java |    24 +
 ...TestQuasiClosedStuckOverReplicationHandler.java |    96 +-
 .../TestQuasiClosedStuckReplicaCount.java          |   302 +-
 ...estQuasiClosedStuckUnderReplicationHandler.java |    61 +-
 .../replication/TestReplicationManager.java        |     6 +-
 .../health/TestEmptyContainerHandler.java          |    36 +
 .../TestQuasiClosedStuckReplicationCheck.java      |    40 +-
 .../container/states/TestContainerStateMap.java    |    76 +
 .../hdds/scm/ha/TestReplicationAnnotation.java     |     7 +-
 .../hadoop/hdds/scm/ha/TestSCMRatisRequest.java    |    19 -
 .../hadoop/hdds/scm/ha/TestSCMRatisResponse.java   |     4 +-
 .../hadoop/hdds/scm/ha/TestSCMStateMachine.java    |    52 +
 ...tegerCodec.java => TestScmBigIntegerCodec.java} |     9 +-
 .../hadoop/hdds/scm/ha/io/TestScmListCodec.java}   |    46 +-
 ...Codec.java => TestScmX509CertificateCodec.java} |    13 +-
 .../hdds/scm/node/TestContainerPlacement.java      |     3 +-
 .../hadoop/hdds/scm/node/TestDeadNodeHandler.java  |   183 +
 .../hdds/scm/node/TestPendingContainerTracker.java |   381 +
 .../hdds/scm/node/states/TestNodeStateMap.java     |     3 +-
 .../hdds/scm/pipeline/MockPipelineManager.java     |    11 +-
 .../hdds/scm/pipeline/TestPipelineManagerImpl.java |    56 +-
 .../scm/pipeline/TestPipelinePlacementFactory.java |     3 +-
 .../safemode/TestHealthyPipelineSafeModeRule.java  |   113 +
 .../TestOneReplicaPipelineSafeModeRule.java        |    44 +
 .../scm/security/TestRootCARotationManager.java    |     2 +-
 .../hadoop/hdds/scm/server/TestSCMCertStore.java   |     3 +-
 .../scm/server/TestSCMClientProtocolServer.java    |    31 +-
 .../ozone/container/common/TestEndPoint.java       |     5 +-
 .../replicationManagerTests/quasi_closed.json      |    57 +-
 hadoop-hdds/test-utils/pom.xml                     |    82 +-
 .../org/apache/ozone/test/GenericTestUtils.java    |     0
 .../java/org/apache/ozone/test/InputSubstream.java |     0
 .../java/org/apache/ozone/test/IntLambda.java      |     0
 .../java/org/apache/ozone/test/JacocoServer.java   |    13 +-
 .../org/apache/ozone/test/LambdaTestUtils.java     |     6 +-
 .../java/org/apache/ozone/test/Log4j1Capturer.java |     0
 .../java/org/apache/ozone/test/Log4j2Capturer.java |     0
 .../java/org/apache/ozone/test/MetricsAsserts.java |     0
 .../java/org/apache/ozone/test/OzoneTestBase.java  |     0
 .../java/org/apache/ozone/test/SpyInputStream.java |     0
 .../org/apache/ozone/test/SpyOutputStream.java     |     0
 .../java/org/apache/ozone/test/TestClock.java      |     0
 .../apache/ozone/test/TimedOutTestsListener.java   |     4 +-
 .../java/org/apache/ozone/test/package-info.java   |     0
 .../java/org/apache/ozone/test/tag/Flaky.java      |     0
 .../java/org/apache/ozone/test/tag/Slow.java       |     0
 .../java/org/apache/ozone/test/tag/Unhealthy.java  |     0
 .../org/apache/ozone/test/tag/package-info.java    |     0
 ...g.junit.platform.launcher.TestExecutionListener |     0
 hadoop-ozone/cli-admin/pom.xml                     |    19 +-
 .../scm/cli/ContainerBalancerStartSubcommand.java  |    17 +-
 .../scm/cli/ContainerBalancerStatusSubcommand.java |    24 +-
 .../hdds/scm/cli/ContainerOperationClient.java     |    43 +-
 .../hdds/scm/cli/SafeModeCheckSubcommand.java      |   190 +-
 .../org/apache/hadoop/hdds/scm/cli/ScmOption.java  |    10 +
 .../scm/cli/container/ContainerIDParameters.java   |    14 +-
 .../hdds/scm/cli/container/InfoSubcommand.java     |     4 +-
 .../hdds/scm/cli/container/ListSubcommand.java     |     9 +-
 .../scm/cli/container/ReconcileSubcommand.java     |    11 +-
 .../hdds/scm/cli/container/ReportSubcommand.java   |    65 +-
 .../datanode/AbstractDiskBalancerSubCommand.java   |    36 +-
 .../hdds/scm/cli/datanode/BasicDatanodeInfo.java   |    54 +-
 .../cli/datanode/DecommissionStatusSubCommand.java |     6 +-
 .../scm/cli/datanode/DecommissionSubCommand.java   |     4 +-
 .../cli/datanode/DiskBalancerReportSubcommand.java |   145 +-
 .../cli/datanode/DiskBalancerStartSubcommand.java  |    29 +-
 .../cli/datanode/DiskBalancerStatusSubcommand.java |    41 +-
 .../cli/datanode/DiskBalancerStopSubcommand.java   |    18 +-
 .../cli/datanode/DiskBalancerSubCommandUtil.java   |    74 +-
 .../cli/datanode/DiskBalancerUpdateSubcommand.java |    33 +-
 .../hdds/scm/cli/datanode/ListInfoSubcommand.java  |    62 +-
 .../scm/cli/datanode/MaintenanceSubCommand.java    |     4 +-
 .../scm/cli/datanode/RecommissionSubCommand.java   |     4 +-
 .../hdds/scm/cli/datanode/UsageInfoSubcommand.java |    14 +-
 .../ozone/admin/nssummary/DiskUsageSubCommand.java |     4 +-
 .../admin/nssummary/FileSizeDistSubCommand.java    |     4 +-
 .../ozone/admin/nssummary/NSSummaryCLIUtils.java   |    13 +-
 .../ozone/admin/om/DecommissionOMSubcommand.java   |     8 +-
 .../ozone/admin/om/FinalizeUpgradeSubCommand.java  |     2 +-
 .../ozone/admin/om/ListOpenFilesSubCommand.java    |    27 +-
 .../org/apache/hadoop/ozone/admin/om/OMAdmin.java  |    18 +-
 .../TestContainerReportSuppressOptions.java        |   310 +
 .../datanode/TestContainerBalancerSubCommand.java  |     7 +-
 .../cli/datanode/TestDiskBalancerSubCommands.java  |   166 +-
 .../scm/cli/datanode/TestListInfoSubcommand.java   |    62 +-
 hadoop-ozone/cli-debug/pom.xml                     |    26 +-
 .../ozone/debug/audit/parser/AuditParser.java      |     4 +-
 .../debug/audit/parser/common/DatabaseHelper.java  |     4 +-
 .../audit/parser/handler/LoadCommandHandler.java   |     6 +-
 .../audit/parser/handler/QueryCommandHandler.java  |     6 +-
 .../parser/handler/TemplateCommandHandler.java     |    16 +-
 .../ozone/debug/kerberos/AuthorizationProbe.java   |   106 +
 .../hadoop/ozone/debug/kerberos/ConfigProbe.java   |   116 +
 .../ozone/debug/kerberos/DiagnoseSubcommand.java   |   113 +
 .../ozone/debug/kerberos/DiagnosticProbe.java}     |    24 +-
 .../ozone/debug/kerberos/EnvironmentProbe.java     |    26 +-
 .../hadoop/ozone/debug/kerberos/HostProbe.java     |    72 +
 .../hadoop/ozone/debug/kerberos/HttpAuthProbe.java |    52 +
 .../ozone/debug/kerberos/JvmKerberosProbe.java     |    98 +
 .../ozone/debug/kerberos/KerberosConfigProbe.java  |    61 +
 .../ozone/debug/kerberos/KerberosSubcommand.java}  |    21 +-
 .../ozone/debug/kerberos/KerberosTicketProbe.java  |    95 +
 .../hadoop/ozone/debug/kerberos/KeytabProbe.java   |    93 +
 .../hadoop/ozone/debug/kerberos/KinitProbe.java    |    61 +
 .../debug/kerberos/PrincipalMappingProbe.java      |   113 +
 .../hadoop/ozone/debug/kerberos/ProbeResult.java}  |    10 +-
 .../ozone/debug/kerberos/SecurityConfigProbe.java  |    72 +
 .../kerberos/TranslatePrincipalSubcommand.java     |    78 +
 .../hadoop/ozone/debug/kerberos}/package-info.java |     4 +-
 .../ozone/debug/om/ContainerToKeyMapping.java      |   118 +-
 .../debug/kerberos/TestOzoneDebugKerberos.java     |   159 +
 .../hadoop/ozone/debug/kerberos}/package-info.java |     4 +-
 .../ozone/debug/om/TestContainerToKeyMapping.java  |   117 +-
 hadoop-ozone/cli-repair/pom.xml                    |    22 +-
 .../schemaupgrade/ContainerUpgradeResult.java      |    36 +-
 .../schemaupgrade/VolumeUpgradeResult.java         |    28 +-
 hadoop-ozone/cli-shell/pom.xml                     |    21 +-
 .../hadoop/ozone/shell/ListPaginationOptions.java  |     2 +-
 .../java/org/apache/hadoop/ozone/shell/Shell.java  |    26 +-
 .../apache/hadoop/ozone/shell/acl/AclOption.java   |     4 +-
 .../hadoop/ozone/shell/bucket/BucketUri.java       |     2 +-
 .../ozone/shell/bucket/DeleteBucketHandler.java    |     4 +-
 .../hadoop/ozone/shell/common/VolumeBucketUri.java |     2 +-
 .../hadoop/ozone/shell/keys/DeleteKeyHandler.java  |     6 +-
 .../org/apache/hadoop/ozone/shell/keys/KeyUri.java |     2 +-
 .../hadoop/ozone/shell/prefix/PrefixUri.java       |     2 +-
 .../shell/snapshot/ListSnapshotDiffHandler.java    |     2 +-
 .../ozone/shell/snapshot/SnapshotDiffHandler.java  |    43 +-
 .../hadoop/ozone/shell/snapshot/SnapshotUri.java   |     2 +-
 .../ozone/shell/tenant/TenantDeleteHandler.java    |     2 +-
 .../ozone/shell/volume/DeleteVolumeHandler.java    |     6 +-
 .../hadoop/ozone/shell/volume/VolumeUri.java       |     2 +-
 hadoop-ozone/client/pom.xml                        |     1 -
 .../apache/hadoop/ozone/client/ObjectStore.java    |    49 +-
 .../apache/hadoop/ozone/client/OzoneBucket.java    |   120 +-
 .../hadoop/ozone/client/OzoneKeyDetails.java       |    16 +
 .../ozone/client/io/KeyDataStreamOutput.java       |     6 +-
 .../ozone/client/protocol/ClientProtocol.java      |   117 +-
 .../apache/hadoop/ozone/client/rpc/RpcClient.java  |   220 +-
 .../hadoop/ozone/client/TestOzoneECClient.java     |     9 +-
 .../common/dev-support/findbugsExcludeFile.xml     |     4 -
 hadoop-ozone/common/pom.xml                        |     5 -
 .../main/java/org/apache/hadoop/ozone/OFSPath.java |    12 +-
 .../main/java/org/apache/hadoop/ozone/OmUtils.java |    31 +-
 .../java/org/apache/hadoop/ozone/OzoneAcl.java     |     8 +-
 .../hadoop/ozone/client/checksum/CrcComposer.java  |    79 +-
 .../hadoop/ozone/client/checksum/CrcUtil.java      |   155 +-
 .../org/apache/hadoop/ozone/om/OMConfigKeys.java   |    17 +-
 .../java/org/apache/hadoop/ozone/om/OmConfig.java  |    20 +-
 .../hadoop/ozone/om/exceptions/OMException.java    |    10 +-
 ...doopRpcOMFollowerReadFailoverProxyProvider.java |   106 +-
 .../ozone/om/ha/OMFailoverProxyProviderBase.java   |    40 +
 .../hadoop/ozone/om/helpers/OMNodeDetails.java     |    17 +-
 .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java  |    32 +
 .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java  |    38 +
 .../ozone/om/helpers/OmKeyLocationInfoGroup.java   |    16 +-
 .../ozone/om/helpers/OmMultipartKeyInfo.java       |   166 +-
 .../ozone/om/helpers/OmMultipartPartInfo.java      |   349 +
 .../ozone/om/helpers/OmMultipartPartKey.java       |   233 +
 .../hadoop/ozone/om/helpers/OzoneFSUtils.java      |    83 +-
 .../hadoop/ozone/om/helpers/OzoneFileStatus.java   |     4 +-
 .../ozone/om/helpers/OzoneFileStatusLight.java     |     4 +-
 .../apache/hadoop/ozone/om/helpers/QuotaUtil.java  |    21 +
 .../hadoop/ozone/om/helpers/ReadConsistency.java   |   107 +
 .../hadoop/ozone/om/helpers/SnapshotDiffJob.java   |    51 +-
 .../ozone/om/protocol/OzoneManagerProtocol.java    |    50 +
 .../ozone/om/protocolPB/GrpcOmTransport.java       |     9 +-
 .../ozone/om/protocolPB/Hadoop3OmTransport.java    |    40 +-
 ...OzoneManagerProtocolClientSideTranslatorPB.java |   104 +-
 .../ozone/security/acl/AssumeRoleRequest.java      |    31 +-
 .../hadoop/ozone/security/acl/RequestContext.java  |    31 +
 .../ozone/snapshot/SnapshotDiffResponse.java       |    59 +-
 .../ozone/snapshot/SubmitSnapshotDiffResponse.java |    65 +
 .../org/apache/hadoop/ozone/util/PayloadUtils.java |     5 +-
 .../org/apache/hadoop/ozone/util/RadixTree.java    |     4 +-
 .../java/org/apache/hadoop/ozone/TestOmUtils.java  |    24 +
 .../ozone/client/checksum/TestCrcComposer.java     |   222 +
 .../hadoop/ozone/client/checksum/TestCrcUtil.java  |   236 +
 ...doopRpcOMFollowerReadFailoverProxyProvider.java |    52 +-
 .../hadoop/ozone/om/helpers/TestOMNodeDetails.java |    24 +
 .../ozone/om/helpers/TestOmMultipartKeyInfo.java   |    41 +
 .../ozone/om/helpers/TestOmMultipartPartInfo.java  |   143 +
 .../ozone/om/helpers/TestOmMultipartPartKey.java   |   204 +
 .../om/helpers/TestOmSnapshotDiffJobCodec.java     |     6 +-
 .../hadoop/ozone/om/helpers/TestOzoneFsUtils.java  |    40 +
 .../hadoop/ozone/om/helpers/TestQuotaUtil.java     |    24 +
 .../ozone/security/acl/TestAssumeRoleRequest.java  |    73 +-
 .../hadoop/ozone/security/acl}/package-info.java   |     4 +-
 .../ozone/snapshot/TestSnapshotDiffResponse.java   |    78 +
 .../snapshot/TestSubmitSnapshotDiffResponse.java   |    97 +
 hadoop-ozone/csi/pom.xml                           |     5 +
 hadoop-ozone/datanode/pom.xml                      |     5 +
 hadoop-ozone/dev-support/checks/install/k3s.sh     |     2 +-
 hadoop-ozone/dist/README.md                        |     6 +-
 .../dist/dev-support/bin/dist-layout-stitching     |     4 +-
 hadoop-ozone/dist/pom.xml                          |    23 +-
 .../Datanode Chunk Read_Write Dashboard.json       |   429 +-
 .../dist/src/main/compose/common/hadoop-test.sh    |     5 +-
 .../dist/src/main/compose/common/ranger.yaml       |    26 +-
 .../src/main/compose/ozone-balancer/test-ec.sh     |     2 +-
 .../src/main/compose/ozone-om-ha/docker-config     |     4 +-
 .../src/main/compose/ozone-topology/docker-config  |     1 +
 .../dist/src/main/compose/ozone/docker-config      |     1 +
 .../src/main/compose/ozonesecure-ha/ranger.yaml    |     4 +-
 .../src/main/compose/ozonesecure-ha/test-ranger.sh |     6 +-
 .../dist/src/main/compose/restart/docker-config    |     1 +
 hadoop-ozone/dist/src/main/compose/testlib.sh      |    28 +-
 .../main/compose/upgrade/compose/ha/docker-config  |     1 +
 .../compose/upgrade/compose/non-ha/docker-config   |     1 +
 hadoop-ozone/dist/src/main/k8s/examples/testlib.sh |     2 +-
 hadoop-ozone/dist/src/main/license/bin/LICENSE.txt |    12 +-
 hadoop-ozone/dist/src/main/license/jar-report.txt  |    14 +-
 .../src/main/smoketest/balancer/testBalancer.robot |    29 +-
 .../dist/src/main/smoketest/cli/classpath.robot    |    47 +-
 .../src/main/smoketest/compatibility/setup.robot   |     9 +-
 .../src/main/smoketest/compatibility/write.robot   |    65 +
 .../omha/data-validation-after-om-bootstrap.robot  |    15 +-
 .../src/main/smoketest/s3/conditionalput.robot     |    89 +
 .../src/main/smoketest/snapshot/snapshot-sh.robot  |    10 +-
 hadoop-ozone/dist/src/shell/ozone/ozone            |    24 +-
 .../dist/src/shell/ozone/ozone-functions.sh        |    45 +-
 .../fault-injection-test/mini-chaos-tests/pom.xml  |     6 -
 .../network-tests/src/test/compose/docker-config   |     4 +-
 hadoop-ozone/freon/pom.xml                         |    95 +-
 .../hadoop/ozone/freon/BaseFreonGenerator.java     |    18 +-
 .../java/org/apache/hadoop/ozone/freon/Freon.java  |     3 +-
 .../hadoop/ozone/freon/OmMetadataGenerator.java    |   143 +-
 .../ozone/freon/OzoneClientKeyGenerator.java       |    15 +-
 .../org/apache/hadoop/ozone/freon/ProgressBar.java |    10 +-
 .../hadoop/ozone/freon/RandomKeyGenerator.java     |    34 +-
 .../hadoop/ozone/freon/S3BucketGenerator.java      |     2 +-
 hadoop-ozone/httpfsgateway/pom.xml                 |    37 +-
 .../src/main/resources/httpfs-default.xml          |     8 -
 .../iceberg/dev-support/findbugsExcludeFile.xml    |    16 +
 hadoop-ozone/iceberg/pom.xml                       |   105 +
 .../ozone/iceberg/RewriteTablePathOzoneAction.java |   254 +
 .../ozone/iceberg/RewriteTablePathOzoneUtils.java  |   116 +
 .../apache/hadoop/ozone/iceberg}/package-info.java |     4 +-
 .../iceberg/TestRewriteTablePathOzoneAction.java   |   279 +
 hadoop-ozone/insight/pom.xml                       |     5 +
 hadoop-ozone/integration-test-recon/pom.xml        |    16 +-
 ...> AbstractTestStorageDistributionEndpoint.java} |   318 +-
 .../ozone/recon/TestReconContainerEndpoint.java    |     9 +-
 .../apache/hadoop/ozone/recon/TestReconTasks.java  |   802 +-
 .../ozone/recon/TestReconTasksMultiNode.java       |   178 +
 .../recon/TestStorageDistributionEndpointEC.java   |    93 +
 .../TestStorageDistributionEndpointRatis.java      |   100 +
 hadoop-ozone/integration-test-s3/pom.xml           |    18 +-
 .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java   |   161 +-
 .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java   |   264 +-
 .../src/test/resources/log4j.properties            |     0
 hadoop-ozone/integration-test/pom.xml              |    26 +-
 .../hadoop/conf/TestConfigurationFieldsBase.java   |    20 +-
 .../fs/contract/AbstractContractConcatTest.java    |     8 +-
 .../AbstractContractContentSummaryTest.java        |     4 +-
 .../AbstractContractCopyFromLocalTest.java         |    10 +-
 .../AbstractContractGetFileStatusTest.java         |    10 +-
 .../AbstractContractLeaseRecoveryTest.java         |    14 +-
 .../AbstractContractMultipartUploaderTest.java     |    55 +-
 .../fs/contract/AbstractContractOpenTest.java      |    74 +-
 .../AbstractContractRootDirectoryTest.java         |    18 +-
 .../fs/ozone/AbstractOzoneFileSystemTest.java      |   390 +-
 .../ozone/AbstractRootedOzoneFileSystemTest.java   |   409 +-
 .../hadoop/fs/ozone/OzoneFileSystemTestBase.java   |   481 +
 .../hadoop/fs/ozone/OzoneFileSystemTestUtils.java  |    47 +
 .../hadoop/fs/ozone/OzoneFileSystemTests.java      |   165 -
 .../hadoop/fs/ozone/TestOzoneFSBucketLayout.java   |     5 +-
 ...oneFileSystemWithStreamingDisabledDatanode.java |   144 +
 .../hadoop/fs/ozone/TestOzoneFsSnapshot.java       |    51 +
 .../scm/TestDatanodeSCMNodesReconfiguration.java   |     8 +-
 .../hadoop/hdds/scm/TestFailoverWithSCMHA.java     |    44 +
 .../hdds/scm/TestSCMDbCheckpointServlet.java       |     8 +-
 .../hdds/scm/TestStorageContainerManager.java      |    13 +-
 .../hadoop/hdds/scm/TestXceiverClientManager.java  |     4 +-
 .../TestReplicationManagerIntegration.java         |   154 +
 .../TestScmDataDistributionFinalization.java       |     6 +-
 .../ozone/TestContainerBalancerOperations.java     |    31 +-
 .../hadoop/ozone/TestOzoneConfigurationFields.java |     6 +-
 .../ozone/client/rpc/OzoneRpcClientTests.java      |   241 +-
 .../rpc/TestContainerStateMachineFailures.java     |    65 +-
 .../ozone/client/rpc/TestOzoneRpcClient.java       |     3 +
 .../ozone/client/rpc/TestSecureOzoneRpcClient.java |     2 +
 .../ozone/freon/TestHadoopDirTreeGenerator.java    |     4 +-
 .../hadoop/ozone/om/TestAddRemoveOzoneManager.java |    76 +-
 .../apache/hadoop/ozone/om/TestOMBootstrap.java    |   202 +
 .../hadoop/ozone/om/TestOMDbCheckpointServlet.java |    12 +-
 .../TestOMDbCheckpointServletInodeBasedXfer.java   |    50 +-
 .../om/TestOMHALeaderSpecificACLEnforcement.java   |    49 +-
 .../hadoop/ozone/om/TestOMRatisSnapshots.java      |   108 +-
 .../org/apache/hadoop/ozone/om/TestOmAcls.java     |     2 +
 .../org/apache/hadoop/ozone/om/TestOmMetrics.java  |     1 +
 .../ozone/om/TestOzoneManagerHAFollowerRead.java   |    32 +
 ...stOzoneManagerHAFollowerReadWithAllRunning.java |   103 +-
 ...OzoneManagerHAFollowerReadWithStoppedNodes.java |     4 +
 .../ozone/om/TestOzoneManagerRestInterface.java    |    16 +-
 .../ozone/om/service/TestRangerBGSyncService.java  |     2 +-
 .../hadoop/ozone/om/snapshot/TestOmSnapshot.java   |     2 +-
 .../om/snapshot/TestOzoneManagerHASnapshot.java    |    80 +
 .../om/snapshot/TestOzoneManagerSnapshotAcl.java   |     4 +
 .../reconfig/TestDatanodeReconfiguration.java      |     2 +
 .../ozone/reconfig/TestOmReconfiguration.java      |    74 +
 .../ozone/reconfig/TestScmReconfiguration.java     |     2 +
 .../scm/node/TestContainerChoosingPolicy.java      |   300 -
 .../node/TestDiskBalancerPolicyPerformance.java    |   381 +
 .../ozone/scm/node/TestVolumeChoosingPolicy.java   |   285 -
 .../ozone/shell/TestDeletedBlocksTxnShell.java     |     3 +-
 .../shell/TestOzoneShellHAWithFollowerRead.java    |    89 +-
 .../hadoop/ozone/shell/TestOzoneTenantShell.java   |     5 +-
 .../ozone/shell/TestSafeModeCheckSubcommandHA.java |   183 +
 .../tools/contract/AbstractContractDistCpTest.java |    14 +-
 .../test/java/org/apache/ozone/test/AclTests.java  |     3 +
 .../src/test/resources/auditlog.properties         |     6 +-
 .../src/test/resources/log4j.properties            |     6 -
 .../src/main/proto/OmClientProtocol.proto          |   120 +-
 hadoop-ozone/interface-storage/pom.xml             |    14 +-
 .../apache/hadoop/ozone/om/OMMetadataManager.java  |     9 +
 .../hadoop/ozone/om/lock/DAGLeveledResource.java   |     0
 .../om/lock/HierarchicalResourceLockManager.java   |     0
 .../hadoop/ozone/om/lock/IOzoneManagerLock.java    |     0
 .../apache/hadoop/ozone/om/lock/LockUsageInfo.java |     0
 .../apache/hadoop/ozone/om/lock/OMLockDetails.java |     0
 .../apache/hadoop/ozone/om/lock/OMLockMetrics.java |     0
 .../apache/hadoop/ozone/om/lock/package-info.java  |     0
 .../ozone/om/lock/TestDAGLeveledResource.java      |     0
 .../apache/hadoop/ozone/om/lock/package-info.java  |     0
 hadoop-ozone/mini-cluster/pom.xml                  |     1 -
 .../hadoop/ozone/MiniOzoneHAClusterImpl.java       |    36 +
 hadoop-ozone/multitenancy-ranger/pom.xml           |    26 +-
 hadoop-ozone/ozone-manager/pom.xml                 |    21 +-
 .../org/apache/hadoop/ozone/audit/OMAction.java    |     3 +-
 .../hadoop/ozone/om/GrpcOzoneManagerServer.java    |     2 +
 .../org/apache/hadoop/ozone/om/KeyManagerImpl.java |    24 +-
 .../org/apache/hadoop/ozone/om/OMDBArchiver.java   |    16 +-
 .../hadoop/ozone/om/OMDBCheckpointServlet.java     |    31 +-
 .../om/OMDBCheckpointServletInodeBasedXfer.java    |   117 +-
 .../java/org/apache/hadoop/ozone/om/OMMetrics.java |    61 +-
 .../hadoop/ozone/om/OMMultiTenantManagerImpl.java  |     4 +
 .../hadoop/ozone/om/OMPerformanceMetrics.java      |    14 +
 .../hadoop/ozone/om/OmMetadataManagerImpl.java     |    50 +-
 .../hadoop/ozone/om/OmSnapshotInternalMetrics.java |    94 +
 .../apache/hadoop/ozone/om/OmSnapshotManager.java  |   104 +-
 .../org/apache/hadoop/ozone/om/OzoneManager.java   |   310 +-
 .../hadoop/ozone/om/OzoneManagerHttpServer.java    |     2 +
 .../hadoop/ozone/om/OzoneManagerStarter.java       |     2 -
 .../java/org/apache/hadoop/ozone/om/S3Batcher.java |     0
 .../apache/hadoop/ozone/om/S3InMemoryCache.java    |     0
 .../org/apache/hadoop/ozone/om/S3SecretCache.java  |     0
 .../apache/hadoop/ozone/om/S3SecretFunction.java   |     0
 .../hadoop/ozone/om/S3SecretLockedManager.java     |     0
 .../apache/hadoop/ozone/om/S3SecretManager.java    |     0
 .../org/apache/hadoop/ozone/om/S3SecretStore.java  |     0
 .../hadoop/ozone/om/SnapshotChainManager.java      |    16 +-
 .../hadoop/ozone/om/SnapshotListJSONServlet.java   |    96 +
 .../hadoop/ozone/om/codec/OMDBDefinition.java      |    28 +-
 .../hadoop/ozone/om/ha/OMPeriodicMetrics.java      |   131 +
 .../ozone/om/lock/DAGResourceLockTracker.java      |     0
 .../ozone/om/lock/LeveledResourceLockTracker.java  |     0
 .../hadoop/ozone/om/lock/OmReadOnlyLock.java       |     0
 .../hadoop/ozone/om/lock/OzoneManagerLock.java     |     0
 .../PoolBasedHierarchicalResourceLockManager.java  |     0
 .../ReadOnlyHierarchicalResourceLockManager.java   |     0
 .../hadoop/ozone/om/lock/ResourceLockTracker.java  |     0
 .../ozone/om/ratis/OzoneManagerRatisServer.java    |    41 +-
 .../ozone/om/ratis/OzoneManagerStateMachine.java   |    95 +-
 .../om/ratis_snapshot/OmRatisSnapshotProvider.java |    18 +-
 .../hadoop/ozone/om/request/OMClientRequest.java   |    39 +-
 .../om/request/bucket/OMBucketDeleteRequest.java   |     2 +-
 .../ozone/om/request/file/OMFileRequest.java       |     4 +-
 .../ozone/om/request/key/OMKeyCommitRequest.java   |    35 +-
 .../om/request/key/OMKeyCommitRequestWithFSO.java  |     1 +
 .../ozone/om/request/key/OMKeyCreateRequest.java   |    66 +-
 .../om/request/key/OMKeyCreateRequestWithFSO.java  |     2 +
 .../hadoop/ozone/om/request/key/OMKeyRequest.java  |     6 +
 .../ozone/om/request/key/OMKeysDeleteRequest.java  |     6 +-
 .../request/snapshot/OMSnapshotCreateRequest.java  |    12 +-
 .../request/snapshot/OMSnapshotDeleteRequest.java  |    12 +-
 .../request/snapshot/OMSnapshotRenameRequest.java  |    22 +-
 .../om/request/upgrade/OMCancelPrepareRequest.java |     2 +-
 .../request/upgrade/OMFinalizeUpgradeRequest.java  |     2 +-
 .../om/request/volume/OMQuotaRepairRequest.java    |     2 +-
 .../hadoop/ozone/om/service/CompactDBService.java  |    70 -
 .../hadoop/ozone/om/service/CompactDBUtil.java     |    75 +
 .../hadoop/ozone/om/service/CompactionService.java |    37 +-
 .../ozone/om/service/DirectoryDeletingService.java |    30 +-
 .../om/service/SnapshotDiffCleanupService.java     |     8 +-
 .../om/snapshot/OMSnapshotDirectoryMetrics.java    |   328 +
 .../om/snapshot/OmSnapshotLocalDataManager.java    |     4 +-
 .../hadoop/ozone/om/snapshot/SnapshotCache.java    |    37 +-
 .../ozone/om/snapshot/SnapshotDiffManager.java     |   504 +-
 .../om/snapshot/SnapshotDiffManagerMXBean.java}    |    19 +-
 .../om/snapshot/db/SnapshotDiffDBDefinition.java   |     2 +-
 .../om/snapshot/defrag/SnapshotDefragService.java  |    82 +-
 .../protocolPB/OMAdminProtocolServerSideImpl.java  |     3 +-
 ...OzoneManagerProtocolServerSideTranslatorPB.java |   149 +-
 .../protocolPB/OzoneManagerRequestHandler.java     |    62 +-
 .../ozone/security/acl/OzoneNativeAuthorizer.java  |    47 +
 .../main/resources/webapps/ozoneManager/index.html |     4 +-
 .../webapps/ozoneManager/om-snapshots.html         |   176 +
 .../resources/webapps/ozoneManager/ozoneManager.js |   155 +
 .../webapps/ozoneManager/ratis-events.html}        |    32 +-
 .../apache/hadoop/ozone/om/TestOMDBArchiver.java   |     7 +-
 ...DBCheckpointServletInodeBasedXferNonLeader.java |    80 +
 .../hadoop/ozone/om/TestOmMetadataManager.java     |     2 +
 .../ozone/om/TestSnapshotListJSONServlet.java      |    57 +
 .../hadoop/ozone/om/lock/TestKeyPathLock.java      |     0
 .../hadoop/ozone/om/lock/TestOzoneManagerLock.java |     0
 ...stPoolBasedHierarchicalResourceLockManager.java |     0
 .../om/ratis/TestOzoneManagerDoubleBuffer.java     |     2 +-
 ...TestOzoneManagerDoubleBufferWithOMResponse.java |     2 +-
 .../om/ratis/TestOzoneManagerStateMachine.java     |  1105 +-
 .../TestOmRatisSnapshotProvider.java               |    12 +-
 .../request/TestOMClientRequestWithUserInfo.java   |     2 +-
 .../ozone/om/request/bucket/TestBucketRequest.java |     2 +-
 .../request/bucket/TestOMBucketCreateRequest.java  |    35 +-
 .../request/file/TestOMDirectoryCreateRequest.java |     2 +-
 .../file/TestOMDirectoryCreateRequestWithFSO.java  |     6 +-
 .../om/request/key/TestOMKeyCommitRequest.java     |    77 +-
 .../om/request/key/TestOMKeyCreateRequest.java     |   237 +
 .../request/key/TestOMKeyCreateRequestWithFSO.java |     4 +-
 .../ozone/om/request/key/TestOMKeyRequest.java     |     4 +-
 .../s3/multipart/TestS3MultipartRequest.java       |     2 +-
 .../s3/security/TestS3GetSecretRequest.java        |     2 +-
 .../s3/tenant/TestOMTenantCreateRequest.java       |     2 +-
 .../s3/tenant/TestOMTenantDeleteRequest.java       |     2 +-
 .../snapshot/TestOMSnapshotCreateRequest.java      |     1 +
 .../snapshot/TestOMSnapshotDeleteRequest.java      |     2 +
 .../snapshot/TestOMSnapshotRenameRequest.java      |    22 +
 .../validation/TestOMValidatorProcessor.java       |    16 +-
 .../om/request/volume/TestOMVolumeRequest.java     |     2 +-
 .../om/service/TestDirectoryDeletingService.java   |    94 +-
 .../ozone/om/service/TestKeyDeletingService.java   |    37 +-
 .../om/service/TestSnapshotDiffCleanupService.java |     4 +-
 .../ozone/om/snapshot/SnapshotTestUtils.java       |    46 +-
 .../snapshot/TestOMSnapshotDirectoryMetrics.java   |   125 +
 .../snapshot/TestOmSnapshotLocalDataManager.java   |    77 +
 .../ozone/om/snapshot/TestSnapshotCache.java       |   143 +-
 .../ozone/om/snapshot/TestSnapshotDiffManager.java |   442 +-
 .../om/snapshot/TestSnapshotDiffManagerMXBean.java |   156 +
 .../snapshot/TestSnapshotRequestAndResponse.java   |     4 +-
 .../defrag/TestInodeMetadataRocksDBCheckpoint.java |    24 +
 .../snapshot/defrag/TestSnapshotDefragService.java |   149 +
 .../protocolPB/TestOzoneManagerRequestHandler.java |   156 +
 .../ozone/security/acl/TestOzoneBlacklist.java     |   235 +
 .../ozone/security/acl/TestRequestContext.java     |    86 +
 .../src/test/resources/junit-platform.properties   |    27 -
 .../hadoop/fs/ozone/BasicOzoneFileSystem.java      |     3 +-
 .../ozone/BasicRootedOzoneClientAdapterImpl.java   |    13 +-
 .../hadoop/fs/ozone/Hadoop27RpcTransport.java      |    35 +-
 hadoop-ozone/ozonefs-shaded/pom.xml                |    67 +-
 hadoop-ozone/pom.xml                               |    10 +
 .../ozone/recon/codegen/TableNamingStrategy.java   |    12 +-
 .../recon/schema/ContainerSchemaDefinition.java    |    20 +-
 hadoop-ozone/recon/pom.xml                         |     8 +-
 .../hadoop/ozone/recon/ReconControllerModule.java  |     7 +
 .../hadoop/ozone/recon/ReconRestServletModule.java |     8 +-
 .../org/apache/hadoop/ozone/recon/ReconServer.java |    47 +-
 .../hadoop/ozone/recon/ReconServerConfigKeys.java  |    32 +
 .../ozone/recon/api/ClusterStateEndpoint.java      |     7 +-
 .../hadoop/ozone/recon/api/ContainerEndpoint.java  |   102 +-
 .../recon/api/StorageDistributionEndpoint.java     |    95 +-
 .../ozone/recon/api/filters/ReconAdminFilter.java  |    34 +-
 .../ozone/recon/api/types/GlobalStorageReport.java |   182 +-
 .../ozone/recon/api/types/OpenKeyBytesInfo.java    |    56 +
 .../ozone/recon/api/types/UsedSpaceBreakDown.java  |    24 +-
 .../ozone/recon/fsck/ContainerHealthTask.java      |   793 +-
 .../fsck/NoOpsContainerReplicaPendingOps.java      |   112 +
 .../ozone/recon/fsck/ReconReplicationManager.java  |   661 ++
 .../recon/fsck/ReconReplicationManagerReport.java  |    66 +
 .../recon/metrics/ContainerHealthMetrics.java      |    86 -
 .../recon/metrics/ContainerHealthTaskMetrics.java  |    75 +
 .../recon/metrics/OzoneManagerSyncMetrics.java     |     2 +-
 .../persistence/ContainerHealthSchemaManager.java  |   511 +-
 .../ozone/recon/scm/ReconDeadNodeHandler.java      |     5 +-
 .../scm/ReconStorageContainerManagerFacade.java    |   128 +-
 .../recon/scm/ReconStorageContainerSyncHelper.java |   113 +
 .../recon/spi/StorageContainerServiceProvider.java |     8 +-
 .../recon/spi/impl/ContainerKeyPrefixCodec.java    |     9 +-
 .../recon/spi/impl/KeyPrefixContainerCodec.java    |    16 +-
 .../impl/StorageContainerServiceProviderImpl.java  |     9 +-
 .../ozone/recon/tasks/ReconTaskControllerImpl.java |     3 +-
 .../ozone/recon/upgrade/ReconLayoutFeature.java    |     4 +-
 ...ntainersStateContainerIdIndexUpgradeAction.java |    86 +
 .../webapps/recon/ozone-recon-web/api/db.json      |  2750 ++++-
 .../recon/ozone-recon-web/api/pagination.js        |    86 +
 .../webapps/recon/ozone-recon-web/api/routes.json  |     5 +
 .../webapps/recon/ozone-recon-web/package.json     |     6 +-
 .../webapps/recon/ozone-recon-web/pnpm-lock.yaml   | 10112 ++++++++++---------
 .../mocks/capacityMocks/capacityResponseMocks.ts   |    18 +-
 .../src/v2/components/plots/insightsFilePlot.tsx   |    32 +-
 .../src/v2/components/select/multiSelect.tsx       |   134 +-
 .../v2/components/select/multiSelectMenuList.tsx   |   160 +
 .../src/v2/components/select/singleSelect.tsx      |     1 +
 .../src/v2/components/tables/bucketsTable.tsx      |     2 +
 .../src/v2/components/tables/containersTable.tsx   |    71 +-
 .../src/v2/components/tables/datanodesTable.tsx    |     1 +
 .../src/v2/components/tables/pipelinesTable.tsx    |     2 +-
 .../src/v2/constants/capacity.constants.tsx        |    19 +-
 .../src/v2/constants/select.constants.tsx          |     7 +-
 .../src/v2/pages/buckets/buckets.tsx               |     9 +-
 .../src/v2/pages/capacity/capacity.less            |    19 +
 .../src/v2/pages/capacity/capacity.tsx             |    81 +-
 .../src/v2/pages/containers/containers.tsx         |   384 +-
 .../src/v2/pages/insights/insights.tsx             |     3 +-
 .../ozone-recon-web/src/v2/types/capacity.types.ts |    21 +-
 .../src/v2/types/container.types.ts                |    37 +-
 .../src/views/datanodes/datanodes.less             |     2 +-
 .../src/views/insights/insights.tsx                |    24 +-
 .../ozone/recon/api/TestClusterStateEndpoint.java  |     2 +-
 .../ozone/recon/api/TestContainerEndpoint.java     |    42 +-
 .../recon/api/TestDeletedKeysSearchEndpoint.java   |     2 -
 .../ozone/recon/api/TestOmDBInsightEndPoint.java   |     2 -
 .../ozone/recon/api/TestOpenContainerCount.java    |     2 -
 .../recon/api/TestOpenKeysSearchEndpoint.java      |     2 -
 .../recon/api/TestStorageDistributionEndpoint.java |    28 +-
 .../ozone/recon/api/TestTriggerDBSyncEndpoint.java |     2 -
 .../ozone/recon/api/filters/TestAdminFilter.java   |    61 +-
 .../ozone/recon/fsck/TestContainerHealthTask.java  |   799 +-
 .../TestContainerHealthTaskRecordGenerator.java    |   710 --
 .../recon/fsck/TestReconReplicationManager.java    |   515 +
 .../ozone/recon/heatmap/TestHeatMapInfo.java       |     2 -
 .../recon/metrics/TestOzoneManagerSyncMetrics.java |    50 +
 .../TestUnhealthyContainersDerbyPerformance.java   |   764 ++
 .../scm/TestReconStorageContainerSyncHelper.java   |   185 +
 ...tOmTableInsightTaskStaleCounterAfterReinit.java |   243 +
 ...ntainersStateContainerIdIndexUpgradeAction.java |   117 +
 hadoop-ozone/s3gateway/pom.xml                     |     5 +
 .../hadoop/ozone/s3/AuthorizationFilter.java       |     7 +-
 .../hadoop/ozone/s3/VirtualHostStyleFilter.java    |     5 +-
 .../endpoint/AuditingBucketOperationHandler.java   |    70 +
 .../hadoop/ozone/s3/endpoint/BucketAclHandler.java |   147 +-
 .../ozone/s3/endpoint/BucketCrudHandler.java       |    60 +-
 .../hadoop/ozone/s3/endpoint/BucketEndpoint.java   |   189 +-
 .../ozone/s3/endpoint/BucketOperationHandler.java  |    14 +-
 .../s3/endpoint/BucketOperationHandlerChain.java   |    95 +
 ...CompleteMultipartUploadRequestUnmarshaller.java |     9 +-
 .../hadoop/ozone/s3/endpoint/EndpointBase.java     |   155 +-
 .../s3/endpoint/ListMultipartUploadsHandler.java   |    92 +
 .../ozone/s3/endpoint/MessageUnmarshaller.java     |     8 +-
 .../endpoint/MultiDeleteRequestUnmarshaller.java   |     2 +
 .../ozone/s3/endpoint/MultipartKeyHandler.java     |     2 -
 .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java   |   253 +-
 .../ozone/s3/endpoint/ObjectEndpointStreaming.java |    32 +-
 .../ozone/s3/endpoint/ObjectTaggingHandler.java    |     9 +-
 .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java |    86 +-
 .../ozone/s3/endpoint/S3ConditionalRequest.java    |   239 +
 .../hadoop/ozone/s3/endpoint/S3RequestContext.java |    78 +
 .../hadoop/ozone/s3/exception/OS3Exception.java    |    37 +-
 .../hadoop/ozone/s3/exception/S3ErrorTable.java    |   264 +-
 .../ozone/s3/signature/AWSSignatureProcessor.java  |     2 +-
 .../ozone/s3/signature/StringToSignProducer.java   |    34 +-
 .../org/apache/hadoop/ozone/s3/util/S3Consts.java  |     7 +
 .../org/apache/hadoop/ozone/s3/util/S3Utils.java   |    34 +-
 .../hadoop/ozone/s3secret/S3SecretAdminFilter.java |     6 +
 .../hadoop/ozone/client/ClientProtocolStub.java    |    63 +
 .../hadoop/ozone/client/ObjectStoreStub.java       |    54 +-
 .../hadoop/ozone/client/OzoneBucketStub.java       |    70 +-
 .../hadoop/ozone/client/OzoneVolumeStub.java       |    16 +-
 .../ozone/s3/TestVirtualHostStyleFilter.java       |     2 +-
 .../ozone/s3/endpoint/EndpointTestUtils.java       |     9 +-
 .../ozone/s3/endpoint/TestBucketAclHandler.java    |   112 +-
 .../hadoop/ozone/s3/endpoint/TestBucketDelete.java |     4 +-
 .../hadoop/ozone/s3/endpoint/TestObjectGet.java    |    88 +
 .../hadoop/ozone/s3/endpoint/TestObjectHead.java   |    85 +
 .../hadoop/ozone/s3/endpoint/TestObjectPut.java    |   118 +-
 .../hadoop/ozone/s3/endpoint/TestPartUpload.java   |     8 +-
 .../ozone/s3/endpoint/TestPermissionCheck.java     |    32 +-
 .../ozone/s3/exception/TestOS3Exceptions.java      |     7 +-
 .../ozone/s3/metrics/TestS3GatewayMetrics.java     |     9 +-
 .../s3/signature/TestAWSSignatureProcessor.java    |    24 +-
 .../apache/hadoop/ozone/s3/util/TestS3Utils.java   |    41 +
 hadoop-ozone/tools/pom.xml                         |    28 +-
 .../hadoop/ozone/local/LocalOzoneRuntime.java}     |    25 +-
 .../org/apache/hadoop/ozone/local/OzoneLocal.java  |    52 +
 .../apache/hadoop/ozone/local}/package-info.java   |     4 +-
 .../apache/hadoop/ozone/local/TestOzoneLocal.java  |   100 +
 .../vapor/dev-support/findbugsExcludeFile.xml      |    16 +
 hadoop-ozone/{cli-repair => vapor}/pom.xml         |   117 +-
 .../hadoop/ozone/freon/BaseAppendLogGenerator.java |     2 +-
 .../hadoop/ozone/freon/ChunkManagerDiskWrite.java  |     4 +-
 .../ozone/freon/ClosedContainerReplicator.java     |     4 +-
 .../ozone/freon/DatanodeSimulationState.java       |     0
 .../hadoop/ozone/freon/DatanodeSimulator.java      |     6 +-
 .../freon/FollowerAppendLogEntryGenerator.java     |     2 +-
 .../ozone/freon/LeaderAppendLogEntryGenerator.java |     2 +-
 .../hadoop/ozone/freon/SCMThroughputBenchmark.java |     4 +-
 .../hadoop/ozone/freon/StreamingGenerator.java     |     4 +-
 .../java/org/apache/hadoop/ozone/freon/Vapor.java  |    23 +-
 .../hadoop/ozone/freon/VaporSubcommand.java}       |    10 +-
 .../freon/containergenerator/BaseGenerator.java    |     3 +-
 .../containergenerator/GeneratorDatanode.java      |    12 +-
 .../freon/containergenerator/GeneratorOm.java      |     4 +-
 .../freon/containergenerator/GeneratorScm.java     |     4 +-
 .../freon/containergenerator/package-info.java     |     0
 .../apache/hadoop/ozone/freon}/package-info.java   |     4 +-
 .../containergenerator/TestGeneratorDatanode.java  |     0
 pom.xml                                            |   344 +-
 .../FileSystem/failure_injector_fs.cc              |     2 +-
 950 files changed, 48530 insertions(+), 17010 deletions(-)

diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index 163696b4f15,7d3f8629f0e..8e91d72d798
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@@ -59,18 -55,17 +59,19 @@@ public enum OzoneManagerVersion impleme
    S3_LIST_MULTIPART_UPLOADS_PAGINATION(11,
        "OzoneManager version that supports S3 list multipart uploads API with pagination"),
  
+   ATOMIC_CREATE_IF_NOT_EXISTS(12,
+       "OzoneManager version that supports explicit create-if-not-exists key semantics"),
 -    
++
 +  ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
 +
    FUTURE_VERSION(-1, "Used internally in the client when the server side is "
        + " newer and an unknown server version has arrived to the client.");
  
-   //////////////////////////////  //////////////////////////////
 -  public static final OzoneManagerVersion CURRENT = latest();
--
 -  private static final Map<Integer, OzoneManagerVersion> BY_PROTO_VALUE =
 +  private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
        Arrays.stream(values())
 -          .collect(toMap(OzoneManagerVersion::toProtoValue, identity()));
 +          .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
 +
 +  public static final OzoneManagerVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey());
  
    private final int version;
    private final String description;
diff --cc hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 1ef451f5a7f,933bb4a0087..1dcecb9008c
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@@ -87,8 -87,8 +87,10 @@@ message ScmContainerLocationRequest 
    optional ContainerBalancerStatusInfoRequestProto containerBalancerStatusInfoRequest = 48;
    optional ReconcileContainerRequestProto reconcileContainerRequest = 49;
    optional GetDeletedBlocksTxnSummaryRequestProto getDeletedBlocksTxnSummaryRequest = 50;
-   optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 51;
-   optional FinalizeUpgradeRequestProto finalizeUpgradeRequest = 52;
+   optional SCMListContainerIDsRequestProto scmListContainerIDsRequest = 51;
+   optional SuppressContainerRequestProto suppressContainerRequest = 52;
++  optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 53;
++  optional FinalizeUpgradeRequestProto finalizeUpgradeRequest = 54;
  }
  
  message ScmContainerLocationResponse {
@@@ -147,8 -147,8 +149,10 @@@
    optional ContainerBalancerStatusInfoResponseProto containerBalancerStatusInfoResponse = 48;
    optional ReconcileContainerResponseProto reconcileContainerResponse = 49;
    optional GetDeletedBlocksTxnSummaryResponseProto getDeletedBlocksTxnSummaryResponse = 50;
-   optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 51;
-   optional FinalizeUpgradeResponseProto finalizeUpgradeResponse = 52;
+   optional SCMListContainerIDsResponseProto scmListContainerIDsResponse = 51;
+   optional SuppressContainerResponseProto suppressContainerResponse = 52;
++  optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 53;
++  optional FinalizeUpgradeResponseProto finalizeUpgradeResponse = 54;
  
    enum Status {
      OK = 1;
@@@ -206,8 -206,8 +210,10 @@@ enum Type 
    GetContainerBalancerStatusInfo = 44;
    ReconcileContainer = 45;
    GetDeletedBlocksTransactionSummary = 46;
-   QueryUpgradeStatus = 47;
-   FinalizeUpgrade = 48;
+   ListContainerIDs = 47;
+   SuppressContainer = 48;
++  QueryUpgradeStatus = 49;
++  FinalizeUpgrade = 50;
  }
  
  /**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index 1e99a49a403,f43ce2c3dc0..45823b5cedf
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@@ -391,6 -425,9 +425,8 @@@ public class SCMStateMachine extends Ba
      // leader ready  in SCMContext.
      scm.getScmContext().setLeaderReady();
      scm.getSCMServiceManager().notifyStatusChanged();
 -    scm.getFinalizationManager().onLeaderReady();
+     addRatisEvent("SCM " + scm.getScmId() +
+         " is ready to serve requests as the leader");
    }
  
    @Override
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index b24bebde3ae,9e4b96999df..eb341cb21d0
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@@ -240,8 -311,8 +242,8 @@@ public class NodeStateManager implement
    }
  
    private DatanodeInfo newDatanodeInfo(DatanodeDetails datanode, LayoutVersionProto layout) {
 -    final NodeStatus status = newNodeStatus(datanode, layout);
 +    final NodeStatus status = newNodeStatus(datanode);
-     return new DatanodeInfo(datanode, status, layout);
+     return new DatanodeInfo(datanode, status, layout, containerRollIntervalMs);
    }
  
    /**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
index 8ffa79c8eb3,c4375e3f20e..f988b773f3b
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
@@@ -53,8 -53,8 +52,7 @@@ import org.apache.hadoop.hdds.scm.ha.Ba
  import org.apache.hadoop.hdds.scm.ha.SCMContext;
  import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
  import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
- import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
  import org.apache.hadoop.hdds.scm.node.NodeManager;
 -import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
  import org.apache.hadoop.hdds.server.events.EventPublisher;
  import org.apache.hadoop.hdds.utils.db.CodecException;
  import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index b761f1c28c4,73bf92e9cd5..2a160b07e40
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@@ -748,18 -756,18 +756,30 @@@ public final class StorageContainerLoca
              .setStatus(Status.OK)
              .setReconcileContainerResponse(reconcileContainer(request.getReconcileContainerRequest()))
              .build();
+       case ListContainerIDs:
+         return ScmContainerLocationResponse.newBuilder()
+             .setCmdType(request.getCmdType())
+             .setStatus(Status.OK)
+             .setScmListContainerIDsResponse(listContainerIDs(request.getScmListContainerIDsRequest()))
+             .build();
+       case SuppressContainer:
+         return ScmContainerLocationResponse.newBuilder()
+             .setCmdType(request.getCmdType())
+             .setStatus(Status.OK)
+             .setSuppressContainerResponse(suppressContainer(request.getSuppressContainerRequest()))
+             .build();
 +      case QueryUpgradeStatus:
 +        return ScmContainerLocationResponse.newBuilder()
 +            .setCmdType(request.getCmdType())
 +            .setStatus(Status.OK)
 +            .setQueryUpgradeStatusResponse(getQueryUpgradeStatus(request.getQueryUpgradeStatusRequest()))
 +            .build();
 +      case FinalizeUpgrade:
 +        impl.finalizeUpgrade();
 +        return ScmContainerLocationResponse.newBuilder()
 +            .setCmdType(request.getCmdType())
 +            .setStatus(Status.OK)
 +            .build();
        default:
          throw new IllegalArgumentException(
              "Unknown command type: " + request.getCmdType());
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
index 3ad70fefa04,3e590013c11..311829af25c
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
@@@ -112,6 -116,17 +112,9 @@@ public class HealthyPipelineSafeModeRul
  
    @Override
    protected synchronized boolean validate() {
 -    boolean shouldRunSafemodeCheck =
 -        FinalizationManager.shouldCreateNewPipelines(
 -            scmContext.getFinalizationCheckpoint());
 -    if (!shouldRunSafemodeCheck) {
 -      LOG.info("All SCM pipelines are closed due to ongoing upgrade " +
 -          "finalization. Bypassing healthy pipeline safemode rule.");
 -      return true;
 -    }
+     if (!validateBasedOnReportProcessing()) {
+       return validateHealthyPipelineSafeModeRuleUsingPipelineManager();
+     }
      return currentHealthyPipelineCount >= healthyPipelineThresholdCount;
    }
  
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
index 18329495c6e,096167b89fc..ed4a07924c6
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
@@@ -24,12 -26,27 +26,12 @@@ import org.apache.hadoop.hdds.utils.db.
  /**
   * Manages the state of finalization in SCM.
   */
- public interface FinalizationStateManager {
+ public interface FinalizationStateManager extends SCMHandler {
  
    @Replicate
 -  void addFinalizingMark() throws IOException;
 -
 -  @Replicate
 -  void removeFinalizingMark() throws IOException;
 -
 -  @Replicate
 -  void finalizeLayoutFeature(Integer layoutVersion)
 +  void finalizeLayoutFeatures(Integer toLayoutVersion)
        throws IOException;
  
 -  /**
 -   * @param query The checkpoint to check for being crossed.
 -   * @return true if SCM's disk state indicates this checkpoint has been
 -   * crossed. False otherwise.
 -   */
 -  boolean crossedCheckpoint(FinalizationCheckpoint query);
 -
 -  FinalizationCheckpoint getFinalizationCheckpoint();
 -
    void setUpgradeContext(SCMUpgradeFinalizationContext context);
  
    /**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
index f7052d0ce0d,ba285cb1d89..16f6cb3b336
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
@@@ -19,7 -19,8 +19,6 @@@ package org.apache.hadoop.hdds.scm.serv
  
  import java.io.IOException;
  import java.util.Objects;
- import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
 -import java.util.concurrent.locks.ReadWriteLock;
 -import java.util.concurrent.locks.ReentrantReadWriteLock;
  import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
  import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
  import org.apache.hadoop.hdds.scm.metadata.Replicate;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
index cbbd52d3c5a,69b1e24dfe3..37e6883c073
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
@@@ -316,8 -328,13 +318,8 @@@ public class MockPipelineManager implem
  
    }
  
 -  @Override
 -  public boolean isPipelineCreationFrozen() {
 -    return false;
 -  }
 -
    @Override
-   public boolean hasEnoughSpace(Pipeline pipeline, long containerSize) {
+   public boolean hasEnoughSpace(Pipeline pipeline) {
      return false;
    }
  
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index f8310902017,bb25b48ec40..f89dae3b3c0
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -320,8 -328,10 +330,9 @@@ import org.apache.hadoop.ozone.snapshot
  import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse;
  import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
  import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+ import org.apache.hadoop.ozone.snapshot.SubmitSnapshotDiffResponse;
  import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
  import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages;
 -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
  import org.apache.hadoop.ozone.util.OzoneNetUtils;
  import org.apache.hadoop.ozone.util.OzoneVersionInfo;
  import org.apache.hadoop.ozone.util.ShutdownHookManager;
@@@ -528,10 -546,17 +546,15 @@@ public final class OzoneManager extend
              .register(OZONE_KEY_DELETING_LIMIT_PER_TASK,
                  this::reconfOzoneKeyDeletingLimitPerTask)
              .register(OZONE_DIR_DELETING_SERVICE_INTERVAL, this::reconfOzoneDirDeletingServiceInterval)
-             .register(OZONE_THREAD_NUMBER_DIR_DELETION, this::reconfOzoneThreadNumberDirDeletion);
+             .register(OZONE_THREAD_NUMBER_DIR_DELETION, this::reconfOzoneThreadNumberDirDeletion)
+             .register(OZONE_BLACKLIST_USERS, this::reconfOzoneBlacklistUsers)
+             .register(OZONE_BLACKLIST_GROUPS, this::reconfOzoneBlacklistGroups)
+             .register(OZONE_READ_BLACKLIST_USERS, this::reconfOzoneReadBlacklistUsers)
+             .register(OZONE_READ_BLACKLIST_GROUPS, this::reconfOzoneReadBlacklistGroups);
  
      reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
+     reconfigurationHandler.registerCompleteCallback(tracingReconfigurationCallback);
  
 -    versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion());
 -    upgradeFinalizer = new OMUpgradeFinalizer(versionManager);
      replicationConfigValidator =
          conf.getObject(ReplicationConfigValidator.class);
  
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
index 136d18c3189,224de13840a..a5b215df34e
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
@@@ -145,8 -149,17 +149,14 @@@ public class TestSnapshotDefragService 
    @Mock
    private IOzoneManagerLock omLock;
  
 -  @Mock
 -  private OMLayoutVersionManager versionManager;
 -
    private DeltaFileComputer deltaFileComputer;
  
+   @Mock
+   private OmSnapshotInternalMetrics snapshotMetrics;
+ 
+   @Mock
+   private OMPerformanceMetrics perfMetrics;
+ 
    @TempDir
    private Path tempDir;
    private OzoneConfiguration configuration;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org