You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2026/05/05 13:22:40 UTC
(ozone) 01/02: Merge branch 'master' into merge-master
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 98d066f5cbecc683813e4e890894987078c80e21
Merge: 4ab7a96b0b3 99b53f44436
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Thu Apr 30 19:01:05 2026 +0100
Merge branch 'master' into merge-master
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
.github/ci.md | 7 -
.github/comment-commands/debug.sh | 21 -
.github/comment-commands/help.sh | 27 -
.github/comment-commands/label.sh | 25 -
.github/comment-commands/retest.sh | 75 -
.github/dependabot.yml | 10 +
.github/process-comment.sh | 56 -
.github/pull_request_template.md | 3 +
.github/workflows/build-ratis.yml | 34 +-
.github/workflows/check.yml | 32 +-
.github/workflows/ci-with-ratis.yml | 9 +-
.github/workflows/ci.yml | 90 +-
.github/workflows/close-stale-prs.yaml | 6 +-
.github/workflows/generate-config-doc.yml | 61 +
.github/workflows/intermittent-test-check.yml | 68 +-
.github/workflows/label-pr.yml | 36 +-
.github/workflows/populate-cache.yml | 14 +-
.github/workflows/post-commit.yml | 7 +-
.github/workflows/pull-request.yml | 8 +-
.github/workflows/repeat-acceptance.yml | 24 +-
.../schedule-label-pr.yml} | 18 +-
.github/workflows/scheduled-cache-update.yml | 3 +-
.github/workflows/update-ozone-site-config-doc.yml | 186 +
.github/workflows/{comments.yaml => zizmor.yml} | 35 +-
CONTRIBUTING.md | 5 +-
README.md | 21 +-
SECURITY.md | 2 +-
dev-support/ci/pr_body_config_doc.sh | 54 +
dev-support/ci/selective_ci_checks.sh | 6 +
dev-support/ci/xml_to_md.py | 177 +
dev-support/pmd/pmd-ruleset.xml | 5 +
hadoop-hdds/cli-common/pom.xml | 16 +-
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 20 +-
hadoop-hdds/client/pom.xml | 1 -
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 186 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 8 +
.../hdds/scm/storage/StreamBlockInputStream.java | 48 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 6 +
.../scm/storage/TestStreamBlockInputStream.java | 193 +-
.../ozone/client/io/TestECBlockInputStream.java | 53 +
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 9 +-
hadoop-hdds/common/pom.xml | 22 +-
.../main/java/com/google/protobuf/ProtoUtils.java | 32 -
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../org/apache/hadoop/hdds/client/BlockID.java | 4 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 11 +
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 10 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 37 +
.../balancer/ContainerBalancerConfiguration.java | 68 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 5 -
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 13 -
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hadoop/hdds/security/SecurityConfig.java | 67 +
.../apache/hadoop/hdds/tracing/LoopSampler.java} | 28 +-
.../apache/hadoop/hdds/tracing/SkipTracing.java | 18 +-
.../apache/hadoop/hdds/tracing/SpanSampler.java | 90 +
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 31 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 128 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 160 +-
.../apache/hadoop/hdds/utils/SimpleStriped.java | 39 +-
.../apache/hadoop/hdds/utils/SlidingWindow.java | 11 +-
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 21 +-
.../hdds/utils/io/RandomAccessFileChannel.java | 37 +-
.../org/apache/hadoop/ipc_/AlignmentContext.java | 2 +-
.../main/java/org/apache/hadoop/ipc_/Server.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 31 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 5 +-
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 21 +
.../org/apache/hadoop/ozone/common/Checksum.java | 4 +-
.../ozone/common/PureJavaCrc32ByteBuffer.java | 12 +
.../ozone/common/PureJavaCrc32CByteBuffer.java | 12 +
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../common/src/main/resources/ozone-default.xml | 243 +-
.../security/TestSecurityConfigTlsSettings.java | 90 +
.../hadoop/hdds/tracing/TestSpanSampling.java | 144 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 34 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 50 +-
.../hadoop/hdds/utils/TestSimpleStriped.java | 14 -
.../hdds/utils/io/TestRandomAccessFileChannel.java | 153 +
hadoop-hdds/config/pom.xml | 10 +
.../hadoop/hdds/conf/ConfigFileAppender.java | 8 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 2 +
hadoop-hdds/container-service/pom.xml | 1 -
.../apache/hadoop/ozone/HddsDatanodeService.java | 27 +-
.../checksum/ContainerChecksumTreeManager.java | 3 +-
.../ozone/container/common/impl/ContainerData.java | 37 +-
.../container/common/impl/ContainerDataYaml.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 200 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 6 +-
.../states/endpoint/VersionEndpointTask.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 2 +
.../server/ratis/ContainerStateMachine.java | 22 +-
.../ozone/container/common/volume/HddsVolume.java | 44 +-
.../container/common/volume/MutableVolumeSet.java | 20 +-
.../container/common/volume/StorageVolume.java | 154 +-
.../common/volume/StorageVolumeChecker.java | 114 +-
.../common/volume/ThrottledAsyncChecker.java | 2 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ContainerChoosingPolicyFactory.java | 55 +
.../diskbalancer/DiskBalancerConfiguration.java | 156 +-
.../container/diskbalancer/DiskBalancerInfo.java | 55 +-
.../diskbalancer/DiskBalancerProtocolServer.java | 5 +-
.../diskbalancer/DiskBalancerService.java | 168 +-
.../container/diskbalancer/DiskBalancerYaml.java | 20 +-
...ChoosingPolicy.java => ContainerCandidate.java} | 42 +-
.../policy/ContainerChoosingPolicy.java | 36 +-
.../policy/DefaultContainerChoosingPolicy.java | 208 +-
.../policy/DefaultVolumeChoosingPolicy.java | 156 -
.../ECReconstructionCoordinator.java | 10 -
.../ozone/container/keyvalue/KeyValueHandler.java | 16 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 27 +-
.../ozone/container/metadata/AbstractRDBStore.java | 14 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 4 +-
.../container/replication/ReplicationServer.java | 2 +
.../ScmHAFinalizeUpgradeActionDatanode.java | 1 +
.../commands/ReplicateContainerCommand.java | 10 +-
.../ozone/container/common/SCMTestUtils.java | 36 +
.../container/common/TestDatanodeStateMachine.java | 11 +-
.../container/common/impl/TestHddsDispatcher.java | 43 +
.../TestReconstructECContainersCommandHandler.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../server/ratis/TestContainerStateMachine.java | 43 +-
.../container/common/volume/TestHddsVolume.java | 39 +
.../common/volume/TestStorageVolumeChecker.java | 185 +
.../volume/TestStorageVolumeHealthChecks.java | 149 +-
.../TestDefaultContainerChoosingPolicy.java | 780 +-
.../TestDefaultVolumeChoosingPolicy.java | 651 --
.../TestDiskBalancerProtocolServer.java | 65 +-
.../diskbalancer/TestDiskBalancerService.java | 81 +-
.../diskbalancer/TestDiskBalancerTask.java | 26 +-
.../diskbalancer/TestDiskBalancerYaml.java | 79 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 51 +
.../TestBackgroundContainerDataScanner.java | 114 +
.../replication/TestReplicationSupervisor.java | 4 +-
.../TestDatanodeUpgradeToContainerIdsTable.java | 12 +-
.../upgrade/TestDatanodeUpgradeToHBaseSupport.java | 14 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 53 +-
.../TestReconstructionECContainersCommands.java | 6 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 28 +-
.../docs/content/design/mpu-gc-optimization.md | 668 ++
.../docs/content/design/s3-conditional-requests.md | 668 +-
.../content/design/storage-distribution-design.jpg | Bin 0 -> 69109 bytes
.../docs/content/design/storage-distribution.md | 427 +
hadoop-hdds/docs/content/feature/DiskBalancer.md | 42 +-
.../docs/content/feature/DiskBalancer.zh.md | 6 +-
.../docs/content/feature/Reconfigurability.md | 28 +-
.../feature/Snapshot-Configuration-Properties.md | 1 +
hadoop-hdds/docs/content/feature/Snapshot.md | 9 +-
.../content/feature/SnapshotDefragmentation.md | 209 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 23 +-
.../erasurecode/rawcoder/TestRawCoderBase.java | 2 +-
hadoop-hdds/framework/pom.xml | 19 +-
.../hadoop/hdds/conf/ReconfigurationHandler.java | 24 +-
.../hdds/conf/TracingReconfigurationCallback.java | 58 +
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 12 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 33 +-
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHandler.java | 18 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 16 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 5 +-
.../hadoop/hdds/scm/net/NodeSchemaManager.java | 12 +
.../apache/hadoop/hdds/scm/net}/package-info.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 49 +-
...inerLocationProtocolClientSideTranslatorPB.java | 137 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 58 +-
.../symmetric/DefaultSecretKeySignerClient.java | 2 +-
.../hdds/security/symmetric/SecretKeyState.java | 9 +-
.../certificate/authority/CertificateStore.java | 8 +-
.../client/DefaultCertificateClient.java | 6 +-
.../certificate/utils/SelfSignedCertificate.java | 2 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 2 +
.../apache/hadoop/hdds/server/OzoneBlacklist.java | 219 +
.../org/apache/hadoop/hdds/server/ServerUtils.java | 39 +
.../org/apache/hadoop/hdds/server/YamlUtils.java | 0
.../hadoop/hdds/server/http/BaseHttpServer.java | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 35 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 38 +-
.../hadoop/hdds/utils/DBCheckpointServlet.java | 14 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 48 +-
.../apache/hadoop/hdds/utils/HttpServletUtils.java | 2 +-
.../utils/db/InodeMetadataRocksDBCheckpoint.java | 83 +-
.../hdds/utils/db/RDBStoreAbstractIterator.java | 19 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 4 +
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 23 +-
.../org/apache/hadoop/ozone/common/BlockGroup.java | 9 +-
.../apache/hadoop/ozone/common/DeletedBlock.java | 17 +-
.../main/resources/network-topology-default.xml | 0
.../main/resources/network-topology-default.yaml | 0
.../main/resources/network-topology-nodegroup.xml | 0
.../src/main/resources/webapps/static/ozone.js | 2 +
.../resources/webapps/static/templates/menu.html | 2 +
.../hdds/fs/TestCachingSpaceUsageSource.java | 35 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 0
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 2 +-
.../hadoop/hdds/scm/net/TestNodeSchemaManager.java | 0
.../hadoop/hdds/scm/net/TestYamlSchemaLoader.java | 0
.../hdds/security/ssl/TestGrpcTlsConfig.java | 253 +
.../apache/hadoop/hdds/server/TestServerUtils.java | 124 +
.../hdds/server/http/TestHttpServer2SSL.java | 305 +
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 18 +-
.../hdds/utils/db/TestFixedLengthStringCodec.java | 7 +-
.../utils/db/TestRDBStoreIteratorWithDBClose.java | 308 +
.../networkTopologyTestFiles/enforce-error.xml | 0
.../networkTopologyTestFiles/external-entity.xml | 0
.../resources/networkTopologyTestFiles/good.xml | 0
.../resources/networkTopologyTestFiles/good.yaml | 0
.../networkTopologyTestFiles/invalid-cost.xml | 0
.../networkTopologyTestFiles/invalid-version.xml | 0
.../networkTopologyTestFiles/middle-leaf.yaml | 0
.../networkTopologyTestFiles/multiple-leaf.xml | 0
.../networkTopologyTestFiles/multiple-root.xml | 0
.../networkTopologyTestFiles/multiple-root.yaml | 0
.../networkTopologyTestFiles/multiple-topology.xml | 0
.../resources/networkTopologyTestFiles/no-leaf.xml | 0
.../resources/networkTopologyTestFiles/no-root.xml | 0
.../networkTopologyTestFiles/no-topology.xml | 0
.../path-layers-size-mismatch.xml | 0
.../path-with-id-reference-failure.xml | 0
.../unknown-layer-type.xml | 0
.../wrong-path-order-1.xml | 0
.../wrong-path-order-2.xml | 0
.../src/main/proto/ScmAdminProtocol.proto | 42 +-
.../interface-client/src/main/proto/hdds.proto | 17 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../utils/db/managed/ManagedRocksIterator.java | 31 +-
hadoop-hdds/rocks-native/pom.xml | 1 -
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 1 -
hadoop-hdds/server-scm/pom.xml | 6 +-
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 5 +
.../scm/block/DeletedBlockLogStateManager.java | 33 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 50 +-
.../container/AbstractContainerReportHandler.java | 31 +-
.../hdds/scm/container/ContainerManager.java | 39 +-
.../hdds/scm/container/ContainerManagerImpl.java | 40 +-
.../hdds/scm/container/ContainerStateManager.java | 36 +-
.../scm/container/ContainerStateManagerImpl.java | 43 +-
.../scm/container/balancer/ContainerBalancer.java | 49 +-
.../ContainerBalancerSelectionCriteria.java | 167 +-
.../container/balancer/ContainerBalancerTask.java | 5 +-
.../hdds/scm/container/balancer/MoveManager.java | 109 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 -
.../container/placement/metrics/SCMMetrics.java | 36 +-
.../replication/ECUnderReplicationHandler.java | 6 +-
...nQueue.java => MonitoringReplicationQueue.java} | 4 +-
.../QuasiClosedStuckOverReplicationHandler.java | 4 +-
.../replication/QuasiClosedStuckReplicaCount.java | 77 +-
.../QuasiClosedStuckUnderReplicationHandler.java | 13 +-
.../replication/RatisUnderReplicationHandler.java | 9 +
.../container/replication/ReplicationManager.java | 74 +-
.../replication/health/EmptyContainerHandler.java | 74 +-
.../health/QuasiClosedStuckReplicationCheck.java | 20 +-
.../scm/container/states/ContainerStateMap.java | 14 +
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 2 +
.../apache/hadoop/hdds/scm/ha/ReflectionUtil.java | 67 -
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 10 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 35 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 38 +-
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 20 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 10 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 7 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 57 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 48 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 8 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 4 +-
.../DeletedBlockLogStateManagerInvoker.java | 148 +
.../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 82 +
.../scm/ha/invoker/ScmInvokerCodeGenerator.java | 533 +
.../hadoop/hdds/scm/ha/invoker}/package-info.java | 5 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 72 -
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 53 -
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 81 -
...igIntegerCodec.java => ScmBigIntegerCodec.java} | 8 +-
.../io/{BooleanCodec.java => ScmBooleanCodec.java} | 10 +-
.../hadoop/hdds/scm/ha/io/ScmByteStringCodec.java | 11 +-
.../hdds/scm/ha/io/{Codec.java => ScmCodec.java} | 12 +-
.../hadoop/hdds/scm/ha/io/ScmCodecFactory.java | 178 +
.../apache/hadoop/hdds/scm/ha/io/ScmEnumCodec.java | 70 +
.../io/{IntegerCodec.java => ScmIntegerCodec.java} | 15 +-
.../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java | 81 +
.../ha/io/{LongCodec.java => ScmLongCodec.java} | 17 +-
...KeyCodec.java => ScmManagedSecretKeyCodec.java} | 10 +-
...Codec.java => ScmNonShadedByteStringCodec.java} | 17 +-
...java => ScmNonShadedGeneratedMessageCodec.java} | 33 +-
.../io/{StringCodec.java => ScmStringCodec.java} | 10 +-
...cateCodec.java => ScmX509CertificateCodec.java} | 14 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 10 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 39 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 13 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hdds/scm/node/PendingContainerTracker.java | 225 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 30 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 24 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 -
.../hdds/scm/pipeline/PipelineStateManager.java | 9 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 4 +-
...inerLocationProtocolServerSideTranslatorPB.java | 73 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 79 +
.../safemode/OneReplicaPipelineSafeModeRule.java | 28 +
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 8 +-
.../hdds/scm/security/RootCARotationHandler.java | 9 +-
.../scm/security/RootCARotationHandlerImpl.java | 4 +-
.../hdds/scm/security/RootCARotationManager.java | 17 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 125 +-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 10 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 20 +-
.../hdds/scm/server/StorageContainerManager.java | 44 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/server/StorageContainerManagerStarter.java | 2 -
.../server/upgrade/FinalizationStateManager.java | 9 +-
.../upgrade/FinalizationStateManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 5 +-
.../main/resources/webapps/scm/ratis-events.html | 32 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../src/main/resources/webapps/scm/scm.js | 26 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 1 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 37 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../scm/container/TestContainerManagerImpl.java | 19 +-
.../scm/container/TestContainerReportHandler.java | 105 +-
.../scm/container/TestContainerStateManager.java | 58 +-
.../hdds/scm/container/balancer/MockedSCM.java | 7 +
.../container/balancer/TestContainerBalancer.java | 37 +
.../TestContainerBalancerDatanodeNodeLimit.java | 17 +
.../TestContainerBalancerSelectionCriteria.java | 320 +
.../balancer/TestContainerBalancerTask.java | 38 +
.../scm/container/balancer/TestMoveManager.java | 175 +
.../algorithms/TestContainerPlacementFactory.java | 2 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 24 +
...TestQuasiClosedStuckOverReplicationHandler.java | 96 +-
.../TestQuasiClosedStuckReplicaCount.java | 302 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 61 +-
.../replication/TestReplicationManager.java | 6 +-
.../health/TestEmptyContainerHandler.java | 36 +
.../TestQuasiClosedStuckReplicationCheck.java | 40 +-
.../container/states/TestContainerStateMap.java | 76 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 19 -
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 4 +-
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 +
...tegerCodec.java => TestScmBigIntegerCodec.java} | 9 +-
.../hadoop/hdds/scm/ha/io/TestScmListCodec.java} | 46 +-
...Codec.java => TestScmX509CertificateCodec.java} | 13 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 183 +
.../hdds/scm/node/TestPendingContainerTracker.java | 381 +
.../hdds/scm/node/states/TestNodeStateMap.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 11 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 56 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 113 +
.../TestOneReplicaPipelineSafeModeRule.java | 44 +
.../scm/security/TestRootCARotationManager.java | 2 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 3 +-
.../scm/server/TestSCMClientProtocolServer.java | 31 +-
.../ozone/container/common/TestEndPoint.java | 5 +-
.../replicationManagerTests/quasi_closed.json | 57 +-
hadoop-hdds/test-utils/pom.xml | 82 +-
.../org/apache/ozone/test/GenericTestUtils.java | 0
.../java/org/apache/ozone/test/InputSubstream.java | 0
.../java/org/apache/ozone/test/IntLambda.java | 0
.../java/org/apache/ozone/test/JacocoServer.java | 13 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 6 +-
.../java/org/apache/ozone/test/Log4j1Capturer.java | 0
.../java/org/apache/ozone/test/Log4j2Capturer.java | 0
.../java/org/apache/ozone/test/MetricsAsserts.java | 0
.../java/org/apache/ozone/test/OzoneTestBase.java | 0
.../java/org/apache/ozone/test/SpyInputStream.java | 0
.../org/apache/ozone/test/SpyOutputStream.java | 0
.../java/org/apache/ozone/test/TestClock.java | 0
.../apache/ozone/test/TimedOutTestsListener.java | 4 +-
.../java/org/apache/ozone/test/package-info.java | 0
.../java/org/apache/ozone/test/tag/Flaky.java | 0
.../java/org/apache/ozone/test/tag/Slow.java | 0
.../java/org/apache/ozone/test/tag/Unhealthy.java | 0
.../org/apache/ozone/test/tag/package-info.java | 0
...g.junit.platform.launcher.TestExecutionListener | 0
hadoop-ozone/cli-admin/pom.xml | 19 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 17 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 24 +-
.../hdds/scm/cli/ContainerOperationClient.java | 43 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 190 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 10 +
.../scm/cli/container/ContainerIDParameters.java | 14 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 4 +-
.../hdds/scm/cli/container/ListSubcommand.java | 9 +-
.../scm/cli/container/ReconcileSubcommand.java | 11 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 65 +-
.../datanode/AbstractDiskBalancerSubCommand.java | 36 +-
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 54 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 6 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 4 +-
.../cli/datanode/DiskBalancerReportSubcommand.java | 145 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 29 +-
.../cli/datanode/DiskBalancerStatusSubcommand.java | 41 +-
.../cli/datanode/DiskBalancerStopSubcommand.java | 18 +-
.../cli/datanode/DiskBalancerSubCommandUtil.java | 74 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 33 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 62 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 4 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 14 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 4 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 4 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 13 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 8 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 2 +-
.../ozone/admin/om/ListOpenFilesSubCommand.java | 27 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 18 +-
.../TestContainerReportSuppressOptions.java | 310 +
.../datanode/TestContainerBalancerSubCommand.java | 7 +-
.../cli/datanode/TestDiskBalancerSubCommands.java | 166 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 62 +-
hadoop-ozone/cli-debug/pom.xml | 26 +-
.../ozone/debug/audit/parser/AuditParser.java | 4 +-
.../debug/audit/parser/common/DatabaseHelper.java | 4 +-
.../audit/parser/handler/LoadCommandHandler.java | 6 +-
.../audit/parser/handler/QueryCommandHandler.java | 6 +-
.../parser/handler/TemplateCommandHandler.java | 16 +-
.../ozone/debug/kerberos/AuthorizationProbe.java | 106 +
.../hadoop/ozone/debug/kerberos/ConfigProbe.java | 116 +
.../ozone/debug/kerberos/DiagnoseSubcommand.java | 113 +
.../ozone/debug/kerberos/DiagnosticProbe.java} | 24 +-
.../ozone/debug/kerberos/EnvironmentProbe.java | 26 +-
.../hadoop/ozone/debug/kerberos/HostProbe.java | 72 +
.../hadoop/ozone/debug/kerberos/HttpAuthProbe.java | 52 +
.../ozone/debug/kerberos/JvmKerberosProbe.java | 98 +
.../ozone/debug/kerberos/KerberosConfigProbe.java | 61 +
.../ozone/debug/kerberos/KerberosSubcommand.java} | 21 +-
.../ozone/debug/kerberos/KerberosTicketProbe.java | 95 +
.../hadoop/ozone/debug/kerberos/KeytabProbe.java | 93 +
.../hadoop/ozone/debug/kerberos/KinitProbe.java | 61 +
.../debug/kerberos/PrincipalMappingProbe.java | 113 +
.../hadoop/ozone/debug/kerberos/ProbeResult.java} | 10 +-
.../ozone/debug/kerberos/SecurityConfigProbe.java | 72 +
.../kerberos/TranslatePrincipalSubcommand.java | 78 +
.../hadoop/ozone/debug/kerberos}/package-info.java | 4 +-
.../ozone/debug/om/ContainerToKeyMapping.java | 118 +-
.../debug/kerberos/TestOzoneDebugKerberos.java | 159 +
.../hadoop/ozone/debug/kerberos}/package-info.java | 4 +-
.../ozone/debug/om/TestContainerToKeyMapping.java | 117 +-
hadoop-ozone/cli-repair/pom.xml | 22 +-
.../schemaupgrade/ContainerUpgradeResult.java | 36 +-
.../schemaupgrade/VolumeUpgradeResult.java | 28 +-
hadoop-ozone/cli-shell/pom.xml | 21 +-
.../hadoop/ozone/shell/ListPaginationOptions.java | 2 +-
.../java/org/apache/hadoop/ozone/shell/Shell.java | 26 +-
.../apache/hadoop/ozone/shell/acl/AclOption.java | 4 +-
.../hadoop/ozone/shell/bucket/BucketUri.java | 2 +-
.../ozone/shell/bucket/DeleteBucketHandler.java | 4 +-
.../hadoop/ozone/shell/common/VolumeBucketUri.java | 2 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 2 +-
.../hadoop/ozone/shell/prefix/PrefixUri.java | 2 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 43 +-
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 2 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 2 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +-
.../hadoop/ozone/shell/volume/VolumeUri.java | 2 +-
hadoop-ozone/client/pom.xml | 1 -
.../apache/hadoop/ozone/client/ObjectStore.java | 49 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 120 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 16 +
.../ozone/client/io/KeyDataStreamOutput.java | 6 +-
.../ozone/client/protocol/ClientProtocol.java | 117 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 220 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 9 +-
.../common/dev-support/findbugsExcludeFile.xml | 4 -
hadoop-ozone/common/pom.xml | 5 -
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +-
.../hadoop/ozone/client/checksum/CrcComposer.java | 79 +-
.../hadoop/ozone/client/checksum/CrcUtil.java | 155 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 17 +-
.../java/org/apache/hadoop/ozone/om/OmConfig.java | 20 +-
.../hadoop/ozone/om/exceptions/OMException.java | 10 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 106 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 40 +
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 17 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 32 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 38 +
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 166 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 349 +
.../ozone/om/helpers/OmMultipartPartKey.java | 233 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 83 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 4 +-
.../ozone/om/helpers/OzoneFileStatusLight.java | 4 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 21 +
.../hadoop/ozone/om/helpers/ReadConsistency.java | 107 +
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 51 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 50 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 9 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 40 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 104 +-
.../ozone/security/acl/AssumeRoleRequest.java | 31 +-
.../hadoop/ozone/security/acl/RequestContext.java | 31 +
.../ozone/snapshot/SnapshotDiffResponse.java | 59 +-
.../ozone/snapshot/SubmitSnapshotDiffResponse.java | 65 +
.../org/apache/hadoop/ozone/util/PayloadUtils.java | 5 +-
.../org/apache/hadoop/ozone/util/RadixTree.java | 4 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 24 +
.../ozone/client/checksum/TestCrcComposer.java | 222 +
.../hadoop/ozone/client/checksum/TestCrcUtil.java | 236 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 52 +-
.../hadoop/ozone/om/helpers/TestOMNodeDetails.java | 24 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 41 +
.../ozone/om/helpers/TestOmMultipartPartInfo.java | 143 +
.../ozone/om/helpers/TestOmMultipartPartKey.java | 204 +
.../om/helpers/TestOmSnapshotDiffJobCodec.java | 6 +-
.../hadoop/ozone/om/helpers/TestOzoneFsUtils.java | 40 +
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 24 +
.../ozone/security/acl/TestAssumeRoleRequest.java | 73 +-
.../hadoop/ozone/security/acl}/package-info.java | 4 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 78 +
.../snapshot/TestSubmitSnapshotDiffResponse.java | 97 +
hadoop-ozone/csi/pom.xml | 5 +
hadoop-ozone/datanode/pom.xml | 5 +
hadoop-ozone/dev-support/checks/install/k3s.sh | 2 +-
hadoop-ozone/dist/README.md | 6 +-
.../dist/dev-support/bin/dist-layout-stitching | 4 +-
hadoop-ozone/dist/pom.xml | 23 +-
.../Datanode Chunk Read_Write Dashboard.json | 429 +-
.../dist/src/main/compose/common/hadoop-test.sh | 5 +-
.../dist/src/main/compose/common/ranger.yaml | 26 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 2 +-
.../src/main/compose/ozone-om-ha/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozonesecure-ha/ranger.yaml | 4 +-
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 6 +-
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 28 +-
.../main/compose/upgrade/compose/ha/docker-config | 1 +
.../compose/upgrade/compose/non-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 12 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 14 +-
.../src/main/smoketest/balancer/testBalancer.robot | 29 +-
.../dist/src/main/smoketest/cli/classpath.robot | 47 +-
.../src/main/smoketest/compatibility/setup.robot | 9 +-
.../src/main/smoketest/compatibility/write.robot | 65 +
.../omha/data-validation-after-om-bootstrap.robot | 15 +-
.../src/main/smoketest/s3/conditionalput.robot | 89 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 24 +-
.../dist/src/shell/ozone/ozone-functions.sh | 45 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 6 -
.../network-tests/src/test/compose/docker-config | 4 +-
hadoop-ozone/freon/pom.xml | 95 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 18 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 143 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 2 +-
hadoop-ozone/httpfsgateway/pom.xml | 37 +-
.../src/main/resources/httpfs-default.xml | 8 -
.../iceberg/dev-support/findbugsExcludeFile.xml | 16 +
hadoop-ozone/iceberg/pom.xml | 105 +
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 254 +
.../ozone/iceberg/RewriteTablePathOzoneUtils.java | 116 +
.../apache/hadoop/ozone/iceberg}/package-info.java | 4 +-
.../iceberg/TestRewriteTablePathOzoneAction.java | 279 +
hadoop-ozone/insight/pom.xml | 5 +
hadoop-ozone/integration-test-recon/pom.xml | 16 +-
...> AbstractTestStorageDistributionEndpoint.java} | 318 +-
.../ozone/recon/TestReconContainerEndpoint.java | 9 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 802 +-
.../ozone/recon/TestReconTasksMultiNode.java | 178 +
.../recon/TestStorageDistributionEndpointEC.java | 93 +
.../TestStorageDistributionEndpointRatis.java | 100 +
hadoop-ozone/integration-test-s3/pom.xml | 18 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 161 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 264 +-
.../src/test/resources/log4j.properties | 0
hadoop-ozone/integration-test/pom.xml | 26 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 20 +-
.../fs/contract/AbstractContractConcatTest.java | 8 +-
.../AbstractContractContentSummaryTest.java | 4 +-
.../AbstractContractCopyFromLocalTest.java | 10 +-
.../AbstractContractGetFileStatusTest.java | 10 +-
.../AbstractContractLeaseRecoveryTest.java | 14 +-
.../AbstractContractMultipartUploaderTest.java | 55 +-
.../fs/contract/AbstractContractOpenTest.java | 74 +-
.../AbstractContractRootDirectoryTest.java | 18 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 390 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 409 +-
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 481 +
.../hadoop/fs/ozone/OzoneFileSystemTestUtils.java | 47 +
.../hadoop/fs/ozone/OzoneFileSystemTests.java | 165 -
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 144 +
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 51 +
.../scm/TestDatanodeSCMNodesReconfiguration.java | 8 +-
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 44 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 8 +-
.../hdds/scm/TestStorageContainerManager.java | 13 +-
.../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +-
.../TestReplicationManagerIntegration.java | 154 +
.../TestScmDataDistributionFinalization.java | 6 +-
.../ozone/TestContainerBalancerOperations.java | 31 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 241 +-
.../rpc/TestContainerStateMachineFailures.java | 65 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 3 +
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +
.../ozone/freon/TestHadoopDirTreeGenerator.java | 4 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 76 +-
.../apache/hadoop/ozone/om/TestOMBootstrap.java | 202 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 12 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 50 +-
.../om/TestOMHALeaderSpecificACLEnforcement.java | 49 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 108 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 2 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 1 +
.../ozone/om/TestOzoneManagerHAFollowerRead.java | 32 +
...stOzoneManagerHAFollowerReadWithAllRunning.java | 103 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 4 +
.../ozone/om/TestOzoneManagerRestInterface.java | 16 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 80 +
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 4 +
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../ozone/reconfig/TestOmReconfiguration.java | 74 +
.../ozone/reconfig/TestScmReconfiguration.java | 2 +
.../scm/node/TestContainerChoosingPolicy.java | 300 -
.../node/TestDiskBalancerPolicyPerformance.java | 381 +
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 -
.../ozone/shell/TestDeletedBlocksTxnShell.java | 3 +-
.../shell/TestOzoneShellHAWithFollowerRead.java | 89 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 5 +-
.../ozone/shell/TestSafeModeCheckSubcommandHA.java | 183 +
.../tools/contract/AbstractContractDistCpTest.java | 14 +-
.../test/java/org/apache/ozone/test/AclTests.java | 3 +
.../src/test/resources/auditlog.properties | 6 +-
.../src/test/resources/log4j.properties | 6 -
.../src/main/proto/OmClientProtocol.proto | 120 +-
hadoop-ozone/interface-storage/pom.xml | 14 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +
.../hadoop/ozone/om/lock/DAGLeveledResource.java | 0
.../om/lock/HierarchicalResourceLockManager.java | 0
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 0
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 0
.../apache/hadoop/ozone/om/lock/OMLockDetails.java | 0
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 0
.../apache/hadoop/ozone/om/lock/package-info.java | 0
.../ozone/om/lock/TestDAGLeveledResource.java | 0
.../apache/hadoop/ozone/om/lock/package-info.java | 0
hadoop-ozone/mini-cluster/pom.xml | 1 -
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 36 +
hadoop-ozone/multitenancy-ranger/pom.xml | 26 +-
hadoop-ozone/ozone-manager/pom.xml | 21 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 2 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 24 +-
.../org/apache/hadoop/ozone/om/OMDBArchiver.java | 16 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 31 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 117 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 61 +-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 4 +
.../hadoop/ozone/om/OMPerformanceMetrics.java | 14 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +-
.../hadoop/ozone/om/OmSnapshotInternalMetrics.java | 94 +
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 104 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 310 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 -
.../java/org/apache/hadoop/ozone/om/S3Batcher.java | 0
.../apache/hadoop/ozone/om/S3InMemoryCache.java | 0
.../org/apache/hadoop/ozone/om/S3SecretCache.java | 0
.../apache/hadoop/ozone/om/S3SecretFunction.java | 0
.../hadoop/ozone/om/S3SecretLockedManager.java | 0
.../apache/hadoop/ozone/om/S3SecretManager.java | 0
.../org/apache/hadoop/ozone/om/S3SecretStore.java | 0
.../hadoop/ozone/om/SnapshotChainManager.java | 16 +-
.../hadoop/ozone/om/SnapshotListJSONServlet.java | 96 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 28 +-
.../hadoop/ozone/om/ha/OMPeriodicMetrics.java | 131 +
.../ozone/om/lock/DAGResourceLockTracker.java | 0
.../ozone/om/lock/LeveledResourceLockTracker.java | 0
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 0
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 0
.../PoolBasedHierarchicalResourceLockManager.java | 0
.../ReadOnlyHierarchicalResourceLockManager.java | 0
.../hadoop/ozone/om/lock/ResourceLockTracker.java | 0
.../ozone/om/ratis/OzoneManagerRatisServer.java | 41 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 95 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 18 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 39 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 35 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 1 +
.../ozone/om/request/key/OMKeyCreateRequest.java | 66 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 12 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 12 +-
.../request/snapshot/OMSnapshotRenameRequest.java | 22 +-
.../om/request/upgrade/OMCancelPrepareRequest.java | 2 +-
.../request/upgrade/OMFinalizeUpgradeRequest.java | 2 +-
.../om/request/volume/OMQuotaRepairRequest.java | 2 +-
.../hadoop/ozone/om/service/CompactDBService.java | 70 -
.../hadoop/ozone/om/service/CompactDBUtil.java | 75 +
.../hadoop/ozone/om/service/CompactionService.java | 37 +-
.../ozone/om/service/DirectoryDeletingService.java | 30 +-
.../om/service/SnapshotDiffCleanupService.java | 8 +-
.../om/snapshot/OMSnapshotDirectoryMetrics.java | 328 +
.../om/snapshot/OmSnapshotLocalDataManager.java | 4 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 37 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 504 +-
.../om/snapshot/SnapshotDiffManagerMXBean.java} | 19 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 2 +-
.../om/snapshot/defrag/SnapshotDefragService.java | 82 +-
.../protocolPB/OMAdminProtocolServerSideImpl.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 149 +-
.../protocolPB/OzoneManagerRequestHandler.java | 62 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 47 +
.../main/resources/webapps/ozoneManager/index.html | 4 +-
.../webapps/ozoneManager/om-snapshots.html | 176 +
.../resources/webapps/ozoneManager/ozoneManager.js | 155 +
.../webapps/ozoneManager/ratis-events.html} | 32 +-
.../apache/hadoop/ozone/om/TestOMDBArchiver.java | 7 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 80 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 2 +
.../ozone/om/TestSnapshotListJSONServlet.java | 57 +
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 0
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 0
...stPoolBasedHierarchicalResourceLockManager.java | 0
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 1105 +-
.../TestOmRatisSnapshotProvider.java | 12 +-
.../request/TestOMClientRequestWithUserInfo.java | 2 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +-
.../request/bucket/TestOMBucketCreateRequest.java | 35 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 77 +-
.../om/request/key/TestOMKeyCreateRequest.java | 237 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 +-
.../s3/security/TestS3GetSecretRequest.java | 2 +-
.../s3/tenant/TestOMTenantCreateRequest.java | 2 +-
.../s3/tenant/TestOMTenantDeleteRequest.java | 2 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 1 +
.../snapshot/TestOMSnapshotDeleteRequest.java | 2 +
.../snapshot/TestOMSnapshotRenameRequest.java | 22 +
.../validation/TestOMValidatorProcessor.java | 16 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +-
.../om/service/TestDirectoryDeletingService.java | 94 +-
.../ozone/om/service/TestKeyDeletingService.java | 37 +-
.../om/service/TestSnapshotDiffCleanupService.java | 4 +-
.../ozone/om/snapshot/SnapshotTestUtils.java | 46 +-
.../snapshot/TestOMSnapshotDirectoryMetrics.java | 125 +
.../snapshot/TestOmSnapshotLocalDataManager.java | 77 +
.../ozone/om/snapshot/TestSnapshotCache.java | 143 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 442 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 156 +
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +-
.../defrag/TestInodeMetadataRocksDBCheckpoint.java | 24 +
.../snapshot/defrag/TestSnapshotDefragService.java | 149 +
.../protocolPB/TestOzoneManagerRequestHandler.java | 156 +
.../ozone/security/acl/TestOzoneBlacklist.java | 235 +
.../ozone/security/acl/TestRequestContext.java | 86 +
.../src/test/resources/junit-platform.properties | 27 -
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 35 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 67 +-
hadoop-ozone/pom.xml | 10 +
.../ozone/recon/codegen/TableNamingStrategy.java | 12 +-
.../recon/schema/ContainerSchemaDefinition.java | 20 +-
hadoop-ozone/recon/pom.xml | 8 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 7 +
.../hadoop/ozone/recon/ReconRestServletModule.java | 8 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 47 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 32 +
.../ozone/recon/api/ClusterStateEndpoint.java | 7 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 102 +-
.../recon/api/StorageDistributionEndpoint.java | 95 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 34 +-
.../ozone/recon/api/types/GlobalStorageReport.java | 182 +-
.../ozone/recon/api/types/OpenKeyBytesInfo.java | 56 +
.../ozone/recon/api/types/UsedSpaceBreakDown.java | 24 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 793 +-
.../fsck/NoOpsContainerReplicaPendingOps.java | 112 +
.../ozone/recon/fsck/ReconReplicationManager.java | 661 ++
.../recon/fsck/ReconReplicationManagerReport.java | 66 +
.../recon/metrics/ContainerHealthMetrics.java | 86 -
.../recon/metrics/ContainerHealthTaskMetrics.java | 75 +
.../recon/metrics/OzoneManagerSyncMetrics.java | 2 +-
.../persistence/ContainerHealthSchemaManager.java | 511 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 128 +-
.../recon/scm/ReconStorageContainerSyncHelper.java | 113 +
.../recon/spi/StorageContainerServiceProvider.java | 8 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 9 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 16 +-
.../impl/StorageContainerServiceProviderImpl.java | 9 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 3 +-
.../ozone/recon/upgrade/ReconLayoutFeature.java | 4 +-
...ntainersStateContainerIdIndexUpgradeAction.java | 86 +
.../webapps/recon/ozone-recon-web/api/db.json | 2750 ++++-
.../recon/ozone-recon-web/api/pagination.js | 86 +
.../webapps/recon/ozone-recon-web/api/routes.json | 5 +
.../webapps/recon/ozone-recon-web/package.json | 6 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 10112 ++++++++++---------
.../mocks/capacityMocks/capacityResponseMocks.ts | 18 +-
.../src/v2/components/plots/insightsFilePlot.tsx | 32 +-
.../src/v2/components/select/multiSelect.tsx | 134 +-
.../v2/components/select/multiSelectMenuList.tsx | 160 +
.../src/v2/components/select/singleSelect.tsx | 1 +
.../src/v2/components/tables/bucketsTable.tsx | 2 +
.../src/v2/components/tables/containersTable.tsx | 71 +-
.../src/v2/components/tables/datanodesTable.tsx | 1 +
.../src/v2/components/tables/pipelinesTable.tsx | 2 +-
.../src/v2/constants/capacity.constants.tsx | 19 +-
.../src/v2/constants/select.constants.tsx | 7 +-
.../src/v2/pages/buckets/buckets.tsx | 9 +-
.../src/v2/pages/capacity/capacity.less | 19 +
.../src/v2/pages/capacity/capacity.tsx | 81 +-
.../src/v2/pages/containers/containers.tsx | 384 +-
.../src/v2/pages/insights/insights.tsx | 3 +-
.../ozone-recon-web/src/v2/types/capacity.types.ts | 21 +-
.../src/v2/types/container.types.ts | 37 +-
.../src/views/datanodes/datanodes.less | 2 +-
.../src/views/insights/insights.tsx | 24 +-
.../ozone/recon/api/TestClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/TestContainerEndpoint.java | 42 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 2 -
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 2 -
.../ozone/recon/api/TestOpenContainerCount.java | 2 -
.../recon/api/TestOpenKeysSearchEndpoint.java | 2 -
.../recon/api/TestStorageDistributionEndpoint.java | 28 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 2 -
.../ozone/recon/api/filters/TestAdminFilter.java | 61 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 799 +-
.../TestContainerHealthTaskRecordGenerator.java | 710 --
.../recon/fsck/TestReconReplicationManager.java | 515 +
.../ozone/recon/heatmap/TestHeatMapInfo.java | 2 -
.../recon/metrics/TestOzoneManagerSyncMetrics.java | 50 +
.../TestUnhealthyContainersDerbyPerformance.java | 764 ++
.../scm/TestReconStorageContainerSyncHelper.java | 185 +
...tOmTableInsightTaskStaleCounterAfterReinit.java | 243 +
...ntainersStateContainerIdIndexUpgradeAction.java | 117 +
hadoop-ozone/s3gateway/pom.xml | 5 +
.../hadoop/ozone/s3/AuthorizationFilter.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 5 +-
.../endpoint/AuditingBucketOperationHandler.java | 70 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 147 +-
.../ozone/s3/endpoint/BucketCrudHandler.java | 60 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 189 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 14 +-
.../s3/endpoint/BucketOperationHandlerChain.java | 95 +
...CompleteMultipartUploadRequestUnmarshaller.java | 9 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 155 +-
.../s3/endpoint/ListMultipartUploadsHandler.java | 92 +
.../ozone/s3/endpoint/MessageUnmarshaller.java | 8 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 2 +
.../ozone/s3/endpoint/MultipartKeyHandler.java | 2 -
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 253 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 32 +-
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 9 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 86 +-
.../ozone/s3/endpoint/S3ConditionalRequest.java | 239 +
.../hadoop/ozone/s3/endpoint/S3RequestContext.java | 78 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 37 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 264 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 2 +-
.../ozone/s3/signature/StringToSignProducer.java | 34 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 7 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 34 +-
.../hadoop/ozone/s3secret/S3SecretAdminFilter.java | 6 +
.../hadoop/ozone/client/ClientProtocolStub.java | 63 +
.../hadoop/ozone/client/ObjectStoreStub.java | 54 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 70 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 16 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../ozone/s3/endpoint/EndpointTestUtils.java | 9 +-
.../ozone/s3/endpoint/TestBucketAclHandler.java | 112 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 88 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 85 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 118 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 8 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 32 +-
.../ozone/s3/exception/TestOS3Exceptions.java | 7 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 9 +-
.../s3/signature/TestAWSSignatureProcessor.java | 24 +-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 41 +
hadoop-ozone/tools/pom.xml | 28 +-
.../hadoop/ozone/local/LocalOzoneRuntime.java} | 25 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 52 +
.../apache/hadoop/ozone/local}/package-info.java | 4 +-
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 100 +
.../vapor/dev-support/findbugsExcludeFile.xml | 16 +
hadoop-ozone/{cli-repair => vapor}/pom.xml | 117 +-
.../hadoop/ozone/freon/BaseAppendLogGenerator.java | 2 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 4 +-
.../ozone/freon/DatanodeSimulationState.java | 0
.../hadoop/ozone/freon/DatanodeSimulator.java | 6 +-
.../freon/FollowerAppendLogEntryGenerator.java | 2 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 4 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 4 +-
.../java/org/apache/hadoop/ozone/freon/Vapor.java | 23 +-
.../hadoop/ozone/freon/VaporSubcommand.java} | 10 +-
.../freon/containergenerator/BaseGenerator.java | 3 +-
.../containergenerator/GeneratorDatanode.java | 12 +-
.../freon/containergenerator/GeneratorOm.java | 4 +-
.../freon/containergenerator/GeneratorScm.java | 4 +-
.../freon/containergenerator/package-info.java | 0
.../apache/hadoop/ozone/freon}/package-info.java | 4 +-
.../containergenerator/TestGeneratorDatanode.java | 0
pom.xml | 344 +-
.../FileSystem/failure_injector_fs.cc | 2 +-
950 files changed, 48530 insertions(+), 17010 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index 163696b4f15,7d3f8629f0e..8e91d72d798
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@@ -59,18 -55,17 +59,19 @@@ public enum OzoneManagerVersion impleme
S3_LIST_MULTIPART_UPLOADS_PAGINATION(11,
"OzoneManager version that supports S3 list multipart uploads API with pagination"),
+ ATOMIC_CREATE_IF_NOT_EXISTS(12,
+ "OzoneManager version that supports explicit create-if-not-exists key semantics"),
-
++
+ ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
+
FUTURE_VERSION(-1, "Used internally in the client when the server side is "
+ " newer and an unknown server version has arrived to the client.");
- ////////////////////////////// //////////////////////////////
- public static final OzoneManagerVersion CURRENT = latest();
--
- private static final Map<Integer, OzoneManagerVersion> BY_PROTO_VALUE =
+ private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
Arrays.stream(values())
- .collect(toMap(OzoneManagerVersion::toProtoValue, identity()));
+ .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+ public static final OzoneManagerVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey());
private final int version;
private final String description;
diff --cc hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 1ef451f5a7f,933bb4a0087..1dcecb9008c
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@@ -87,8 -87,8 +87,10 @@@ message ScmContainerLocationRequest
optional ContainerBalancerStatusInfoRequestProto containerBalancerStatusInfoRequest = 48;
optional ReconcileContainerRequestProto reconcileContainerRequest = 49;
optional GetDeletedBlocksTxnSummaryRequestProto getDeletedBlocksTxnSummaryRequest = 50;
- optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 51;
- optional FinalizeUpgradeRequestProto finalizeUpgradeRequest = 52;
+ optional SCMListContainerIDsRequestProto scmListContainerIDsRequest = 51;
+ optional SuppressContainerRequestProto suppressContainerRequest = 52;
++ optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 53;
++ optional FinalizeUpgradeRequestProto finalizeUpgradeRequest = 54;
}
message ScmContainerLocationResponse {
@@@ -147,8 -147,8 +149,10 @@@
optional ContainerBalancerStatusInfoResponseProto containerBalancerStatusInfoResponse = 48;
optional ReconcileContainerResponseProto reconcileContainerResponse = 49;
optional GetDeletedBlocksTxnSummaryResponseProto getDeletedBlocksTxnSummaryResponse = 50;
- optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 51;
- optional FinalizeUpgradeResponseProto finalizeUpgradeResponse = 52;
+ optional SCMListContainerIDsResponseProto scmListContainerIDsResponse = 51;
+ optional SuppressContainerResponseProto suppressContainerResponse = 52;
++ optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 53;
++ optional FinalizeUpgradeResponseProto finalizeUpgradeResponse = 54;
enum Status {
OK = 1;
@@@ -206,8 -206,8 +210,10 @@@ enum Type
GetContainerBalancerStatusInfo = 44;
ReconcileContainer = 45;
GetDeletedBlocksTransactionSummary = 46;
- QueryUpgradeStatus = 47;
- FinalizeUpgrade = 48;
+ ListContainerIDs = 47;
+ SuppressContainer = 48;
++ QueryUpgradeStatus = 49;
++ FinalizeUpgrade = 50;
}
/**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index 1e99a49a403,f43ce2c3dc0..45823b5cedf
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@@ -391,6 -425,9 +425,8 @@@ public class SCMStateMachine extends Ba
// leader ready in SCMContext.
scm.getScmContext().setLeaderReady();
scm.getSCMServiceManager().notifyStatusChanged();
- scm.getFinalizationManager().onLeaderReady();
+ addRatisEvent("SCM " + scm.getScmId() +
+ " is ready to serve requests as the leader");
}
@Override
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index b24bebde3ae,9e4b96999df..eb341cb21d0
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@@ -240,8 -311,8 +242,8 @@@ public class NodeStateManager implement
}
private DatanodeInfo newDatanodeInfo(DatanodeDetails datanode, LayoutVersionProto layout) {
- final NodeStatus status = newNodeStatus(datanode, layout);
+ final NodeStatus status = newNodeStatus(datanode);
- return new DatanodeInfo(datanode, status, layout);
+ return new DatanodeInfo(datanode, status, layout, containerRollIntervalMs);
}
/**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
index 8ffa79c8eb3,c4375e3f20e..f988b773f3b
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
@@@ -53,8 -53,8 +52,7 @@@ import org.apache.hadoop.hdds.scm.ha.Ba
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
- import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.utils.db.CodecException;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index b761f1c28c4,73bf92e9cd5..2a160b07e40
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@@ -748,18 -756,18 +756,30 @@@ public final class StorageContainerLoca
.setStatus(Status.OK)
.setReconcileContainerResponse(reconcileContainer(request.getReconcileContainerRequest()))
.build();
+ case ListContainerIDs:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setScmListContainerIDsResponse(listContainerIDs(request.getScmListContainerIDsRequest()))
+ .build();
+ case SuppressContainer:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setSuppressContainerResponse(suppressContainer(request.getSuppressContainerRequest()))
+ .build();
+ case QueryUpgradeStatus:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setQueryUpgradeStatusResponse(getQueryUpgradeStatus(request.getQueryUpgradeStatusRequest()))
+ .build();
+ case FinalizeUpgrade:
+ impl.finalizeUpgrade();
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
index 3ad70fefa04,3e590013c11..311829af25c
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
@@@ -112,6 -116,17 +112,9 @@@ public class HealthyPipelineSafeModeRul
@Override
protected synchronized boolean validate() {
- boolean shouldRunSafemodeCheck =
- FinalizationManager.shouldCreateNewPipelines(
- scmContext.getFinalizationCheckpoint());
- if (!shouldRunSafemodeCheck) {
- LOG.info("All SCM pipelines are closed due to ongoing upgrade " +
- "finalization. Bypassing healthy pipeline safemode rule.");
- return true;
- }
+ if (!validateBasedOnReportProcessing()) {
+ return validateHealthyPipelineSafeModeRuleUsingPipelineManager();
+ }
return currentHealthyPipelineCount >= healthyPipelineThresholdCount;
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
index 18329495c6e,096167b89fc..ed4a07924c6
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java
@@@ -24,12 -26,27 +26,12 @@@ import org.apache.hadoop.hdds.utils.db.
/**
* Manages the state of finalization in SCM.
*/
- public interface FinalizationStateManager {
+ public interface FinalizationStateManager extends SCMHandler {
@Replicate
- void addFinalizingMark() throws IOException;
-
- @Replicate
- void removeFinalizingMark() throws IOException;
-
- @Replicate
- void finalizeLayoutFeature(Integer layoutVersion)
+ void finalizeLayoutFeatures(Integer toLayoutVersion)
throws IOException;
- /**
- * @param query The checkpoint to check for being crossed.
- * @return true if SCM's disk state indicates this checkpoint has been
- * crossed. False otherwise.
- */
- boolean crossedCheckpoint(FinalizationCheckpoint query);
-
- FinalizationCheckpoint getFinalizationCheckpoint();
-
void setUpgradeContext(SCMUpgradeFinalizationContext context);
/**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
index f7052d0ce0d,ba285cb1d89..16f6cb3b336
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java
@@@ -19,7 -19,8 +19,6 @@@ package org.apache.hadoop.hdds.scm.serv
import java.io.IOException;
import java.util.Objects;
- import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
index cbbd52d3c5a,69b1e24dfe3..37e6883c073
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
@@@ -316,8 -328,13 +318,8 @@@ public class MockPipelineManager implem
}
- @Override
- public boolean isPipelineCreationFrozen() {
- return false;
- }
-
@Override
- public boolean hasEnoughSpace(Pipeline pipeline, long containerSize) {
+ public boolean hasEnoughSpace(Pipeline pipeline) {
return false;
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index f8310902017,bb25b48ec40..f89dae3b3c0
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -320,8 -328,10 +330,9 @@@ import org.apache.hadoop.ozone.snapshot
import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse;
import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+ import org.apache.hadoop.ozone.snapshot.SubmitSnapshotDiffResponse;
import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
import org.apache.hadoop.ozone.util.ShutdownHookManager;
@@@ -528,10 -546,17 +546,15 @@@ public final class OzoneManager extend
.register(OZONE_KEY_DELETING_LIMIT_PER_TASK,
this::reconfOzoneKeyDeletingLimitPerTask)
.register(OZONE_DIR_DELETING_SERVICE_INTERVAL, this::reconfOzoneDirDeletingServiceInterval)
- .register(OZONE_THREAD_NUMBER_DIR_DELETION, this::reconfOzoneThreadNumberDirDeletion);
+ .register(OZONE_THREAD_NUMBER_DIR_DELETION, this::reconfOzoneThreadNumberDirDeletion)
+ .register(OZONE_BLACKLIST_USERS, this::reconfOzoneBlacklistUsers)
+ .register(OZONE_BLACKLIST_GROUPS, this::reconfOzoneBlacklistGroups)
+ .register(OZONE_READ_BLACKLIST_USERS, this::reconfOzoneReadBlacklistUsers)
+ .register(OZONE_READ_BLACKLIST_GROUPS, this::reconfOzoneReadBlacklistGroups);
reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
+ reconfigurationHandler.registerCompleteCallback(tracingReconfigurationCallback);
- versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion());
- upgradeFinalizer = new OMUpgradeFinalizer(versionManager);
replicationConfigValidator =
conf.getObject(ReplicationConfigValidator.class);
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
index 136d18c3189,224de13840a..a5b215df34e
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/defrag/TestSnapshotDefragService.java
@@@ -145,8 -149,17 +149,14 @@@ public class TestSnapshotDefragService
@Mock
private IOzoneManagerLock omLock;
- @Mock
- private OMLayoutVersionManager versionManager;
-
private DeltaFileComputer deltaFileComputer;
+ @Mock
+ private OmSnapshotInternalMetrics snapshotMetrics;
+
+ @Mock
+ private OMPerformanceMetrics perfMetrics;
+
@TempDir
private Path tempDir;
private OzoneConfiguration configuration;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org