You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by te...@apache.org on 2025/08/12 04:53:53 UTC
(ozone) branch HDDS-5713 updated: Merge master into HDDS-5713 (#8925)
This is an automated email from the ASF dual-hosted git repository.
tejaskriya pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-5713 by this push:
new 261de892c12 Merge master into HDDS-5713 (#8925)
261de892c12 is described below
commit 261de892c1280d748b7263129124ba776d0d86cd
Author: Gargi Jaiswal <13...@users.noreply.github.com>
AuthorDate: Tue Aug 12 10:23:47 2025 +0530
Merge master into HDDS-5713 (#8925)
---
CONTRIBUTING.md | 4 +
.../byteman/container-state-template.btm | 22 +-
.../byteman/fail-create-bucket.btm | 24 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 3 +-
.../hadoop/hdds/utils/BackgroundService.java | 69 ++-
.../apache/hadoop/hdds/utils/SlidingWindow.java | 179 +++++++
.../hadoop/hdds/utils/db/DelegatedCodec.java | 11 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 17 +
.../common/src/main/resources/ozone-default.xml | 66 ++-
.../hadoop/hdds/utils/TestSlidingWindow.java | 141 +++++
.../apache/hadoop/util/TestBackgroundService.java | 167 ++++++
.../checksum/ContainerChecksumTreeManager.java | 97 ++--
.../container/checksum/ContainerDiffReport.java | 10 +-
.../checksum/ContainerMerkleTreeWriter.java | 5 +-
.../ozone/container/common/impl/ContainerData.java | 14 +-
.../ozone/container/common/impl/ContainerSet.java | 77 +--
.../container/common/impl/HddsDispatcher.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 8 +-
.../SetNodeOperationalStateCommandHandler.java | 16 +-
.../ozone/container/common/volume/HddsVolume.java | 1 +
.../container/common/volume/MutableVolumeSet.java | 4 +
.../common/volume/StorageVolumeChecker.java | 66 ++-
.../common/volume/StorageVolumeScannerMetrics.java | 119 +++++
.../container/common/volume/VolumeInfoMetrics.java | 23 +
.../container/keyvalue/KeyValueContainerData.java | 5 +
.../ozone/container/keyvalue/KeyValueHandler.java | 83 ++-
.../container/keyvalue/TarContainerPacker.java | 35 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 36 +-
.../container/metadata/ContainerCreateInfo.java | 76 +++
.../metadata/WitnessedContainerDBDefinition.java | 17 +-
.../metadata/WitnessedContainerMetadataStore.java | 2 +-
.../WitnessedContainerMetadataStoreImpl.java | 55 +-
.../BackgroundContainerMetadataScanner.java | 19 +-
.../container/ozoneimpl/ContainerScanHelper.java | 56 +-
...aScanner.java => OnDemandContainerScanner.java} | 41 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 14 +-
.../container/replication/ContainerImporter.java | 5 +-
.../ContainerTableSchemaFinalizeAction.java | 97 ++++
.../commands/ReconcileContainerCommand.java | 5 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 +
.../checksum/ContainerMerkleTreeTestUtils.java | 45 +-
.../checksum/TestContainerChecksumTreeManager.java | 39 +-
.../checksum/TestContainerMerkleTreeWriter.java | 105 +++-
.../checksum/TestReconcileContainerTask.java | 4 +-
.../common/TestKeyValueContainerData.java | 29 ++
.../common/impl/ContainerImplTestUtils.java | 12 +-
.../container/common/impl/TestContainerSet.java | 25 +-
.../TestDeleteBlocksCommandHandler.java | 25 +
.../ozone/container/common/utils/package-info.java | 12 +-
.../container/common/volume/TestHddsVolume.java | 2 +
.../common/volume/TestPeriodicVolumeChecker.java | 63 ++-
.../common/volume/TestStorageVolumeChecker.java | 47 ++
...stContainerReconciliationWithMockDatanodes.java | 27 +-
.../container/keyvalue/TestKeyValueContainer.java | 65 +++
.../keyvalue/TestKeyValueContainerCheck.java | 6 +-
.../container/keyvalue/TestKeyValueHandler.java | 19 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 4 +-
.../container/keyvalue/TestTarContainerPacker.java | 86 ++--
.../keyvalue/impl/AbstractTestChunkManager.java | 4 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 27 +-
.../TestBackgroundContainerDataScanner.java | 15 +
.../TestBackgroundContainerMetadataScanner.java | 17 +
.../container/ozoneimpl/TestContainerReader.java | 164 ++++++
.../ozoneimpl/TestContainerScannersAbstract.java | 8 +
...nner.java => TestOnDemandContainerScanner.java} | 76 ++-
.../replication/TestContainerImporter.java | 117 +++--
.../replication/TestReplicationSupervisor.java | 38 ++
.../TestDatanodeUpgradeToContainerIdsTable.java | 193 +++++++
hadoop-hdds/docs/content/_index.md | 103 +++-
hadoop-hdds/docs/content/concept/Comparison.md | 77 +++
hadoop-hdds/docs/content/concept/OzoneManager.md | 16 +-
.../content/concept/StorageContainerManager.md | 21 +
.../docs/content/feature/ContainerBalancer.md | 108 ++++
hadoop-hdds/docs/content/feature/Maintenance.md | 78 +++
hadoop-hdds/docs/content/feature/OM-HA.md | 78 +++
hadoop-hdds/docs/content/feature/SCM-HA.md | 93 +++-
.../feature/om-bootstrapping-with-snapshots.md | 148 ++++++
hadoop-hdds/docs/content/integration/DistCp.md | 2 +
hadoop-hdds/docs/content/interface/Ofs.md | 158 ++++++
hadoop-hdds/docs/content/interface/S3.md | 75 ++-
.../docs/content/start/ProductionDeployment.md | 89 ++++
.../docs/content/start/ProductionDeployment.zh.md | 69 +++
hadoop-hdds/docs/content/tools/LogsInOzone.md | 81 ++-
hadoop-hdds/docs/content/tools/Repair.md | 252 +++++++++
.../hadoop/hdds/server/http/HttpServer2.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointServlet.java | 3 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 56 +-
.../org/apache/hadoop/hdds/utils/LogLevel.java | 416 +++++++++++++++
.../hadoop/hdds/utils/RDBSnapshotProvider.java | 9 +-
.../hadoop/hdds/utils/MapBackedTableIterator.java | 12 +-
.../hadoop/hdds/utils/TestRDBSnapshotProvider.java | 18 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../utils/db/managed/ManagedConfigOptions.java | 22 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 22 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 2 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 30 +-
.../ozone/rocksdiff/SSTFilePruningMetrics.java | 135 +++++
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 15 +-
.../block/DatanodeDeletedBlockTransactions.java | 8 +
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 35 +-
.../container/replication/ContainerReplicaOp.java | 18 +-
.../replication/ContainerReplicaPendingOps.java | 98 +++-
.../container/replication/ReplicationManager.java | 26 +-
.../replication/ReplicationManagerUtil.java | 51 ++
.../hdds/scm/safemode/ECContainerSafeModeRule.java | 1 -
.../hdds/scm/safemode/SCMSafeModeManager.java | 4 +-
.../hdds/scm/safemode/SafeModeRuleFactory.java | 14 +-
.../hdds/scm/server/StorageContainerManager.java | 7 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 37 ++
.../TestContainerReplicaPendingOps.java | 197 +++++--
.../replication/TestECUnderReplicationHandler.java | 1 +
.../replication/TestMisReplicationHandler.java | 1 +
...estQuasiClosedStuckUnderReplicationHandler.java | 1 +
.../TestRatisUnderReplicationHandler.java | 1 +
.../replication/TestReplicationManager.java | 8 +-
.../TestReplicationManagerScenarios.java | 3 +-
.../replication/TestReplicationManagerUtil.java | 122 +++++
.../scm/safemode/TestECContainerSafeModeRule.java | 219 ++++++++
.../safemode/TestRatisContainerSafeModeRule.java | 207 ++++++++
.../hdds/scm/safemode/TestSafeModeRuleFactory.java | 11 +-
.../scm/cli/container/ReconcileSubcommand.java | 2 +-
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 165 ++++++
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 139 +++--
.../ozone/admin/om/UpdateRangerSubcommand.java | 2 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 148 +++++-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 5 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 13 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 4 +-
.../ozone/security/OzoneTokenIdentifier.java | 4 +-
hadoop-ozone/dev-support/checks/sonar.sh | 4 +-
.../dashboards/Ozone - DeleteKey Metrics.json | 6 +-
.../Ozone - DeleteKeyProgress Metrics.json | 4 +-
.../dashboards/Ozone - OMComittedIndexMetrics.json | 10 +-
.../dashboards/Ozone - Ozone Manager RocksDB.json | 194 +++----
.../dist/src/main/compose/common/replicas-test.sh | 3 +
.../dist/src/main/compose/common/security.conf | 3 +
.../dist/src/main/compose/ozone-om-ha/startOM.sh | 6 +-
.../src/main/compose/ozonesecure-ha/docker-config | 7 +-
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 6 +-
.../compose/ozonesecure-ha/test-repair-tools.sh | 56 ++
.../dist/src/main/compose/ozonesecure-ha/test.sh | 3 +
.../src/main/k8s/definitions/ozone/config.yaml | 2 +
.../src/main/k8s/definitions/ozone/s3g-ss.yaml | 2 +-
.../examples/getting-started/config-configmap.yaml | 2 +
.../examples/getting-started/s3g-statefulset.yaml | 2 +-
.../k8s/examples/minikube/config-configmap.yaml | 2 +
.../k8s/examples/minikube/s3g-statefulset.yaml | 2 +-
.../k8s/examples/ozone-dev/config-configmap.yaml | 2 +
.../k8s/examples/ozone-dev/s3g-statefulset.yaml | 2 +-
.../k8s/examples/ozone-ha/config-configmap.yaml | 2 +
.../k8s/examples/ozone-ha/s3g-statefulset.yaml | 2 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 2 +
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 2 +-
.../src/main/smoketest/admincli/datanode.robot | 3 +-
.../{kinit.robot => admincli/lib.resource} | 20 +-
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
.../smoketest/{kinit.robot => cli/daemonlog.robot} | 22 +-
.../smoketest/debug/container-state-verifier.robot | 58 +++
.../smoketest/debug/ozone-debug-keywords.robot | 4 +
hadoop-ozone/dist/src/main/smoketest/kinit.robot | 4 +-
.../omha/data-validation-after-om-bootstrap.robot | 2 +-
.../src/main/smoketest/repair/om-compact.robot | 8 +-
.../ratis-transaction-repair.robot} | 21 +-
.../dist/src/main/smoketest/s3/freon.robot | 13 +
.../dist/src/main/smoketest/s3/objectlist.robot | 6 +-
.../{kinit.robot => scmha/container-create.robot} | 18 +-
.../smoketest/security/ozone-secure-tenant.robot | 74 +--
.../dist/src/main/smoketest/upgrade/lib.robot | 16 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 2 +-
.../ozone/freon/ClosedContainerReplicator.java | 2 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 2 +-
.../httpfsgateway/src/main/conf/httpfs-env.sh | 6 +-
.../main/libexec/shellprofile.d/hadoop-httpfs.sh | 54 --
hadoop-ozone/integration-test-recon/pom.xml | 6 +
.../ozone/recon/TestNSSummaryMemoryLeak.java | 502 ++++++++++++++++++
.../apache/hadoop/ozone/recon/TestReconTasks.java | 17 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
hadoop-ozone/integration-test-s3/pom.xml | 40 ++
.../hadoop/ozone/s3/LoadBalanceStrategy.java} | 19 +-
.../hadoop/ozone/s3/MultiS3GatewayService.java | 99 ++++
.../org/apache/hadoop/ozone/s3/ProxyServer.java | 215 ++++++++
.../ozone/s3/ProxyServerIntegrationTest.java | 163 ++++++
.../hadoop/ozone/s3/RoundRobinStrategy.java} | 28 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 60 ++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 113 +++-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 3 +-
.../apache/hadoop/hdds/scm/TestCloseContainer.java | 20 +-
.../hadoop/hdds/scm/TestContainerOperations.java | 9 +-
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 4 +
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 5 +
.../org/apache/hadoop/ozone/TestBlockTokens.java | 4 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 29 +-
.../ozone/client/TestOzoneClientFactory.java | 68 ---
.../ozone/client/rpc/OzoneRpcClientTests.java | 1 +
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 19 +
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 14 -
.../TestContainerCommandReconciliation.java | 51 +-
...tBackgroundContainerDataScannerIntegration.java | 36 +-
.../TestContainerScannerIntegrationAbstract.java | 19 +
...> TestOnDemandContainerScannerIntegration.java} | 51 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 3 -
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 9 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 307 +++++++++--
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 40 +-
.../hadoop/ozone/om/snapshot/SnapshotTests.java | 82 +++
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +-
.../om/snapshot/TestOmSnapshotFileSystem.java | 121 ++---
...stOmSnapshotFileSystemFsoWithLinkedBuckets.java | 27 -
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 2 +
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 3 +
.../TestOmSnapshotWithBucketLinkingLegacy.java | 3 +
.../om/snapshot/TestOzoneManagerHASnapshot.java | 21 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 2 +
...edBuckets.java => TestOzoneSnapshotsNonHA.java} | 17 +-
.../TestSnapshotDirectoryCleaningService.java | 73 ++-
.../ozone/reconfig/TestOmReconfiguration.java | 63 +++
.../org/apache/ozone/test/ClusterForTests.java | 6 +
.../src/main/proto/OmClientProtocol.proto | 19 +
hadoop-ozone/ozone-manager/pom.xml | 4 +
.../hadoop/ozone/om/DeletingServiceMetrics.java | 127 +++++
.../org/apache/hadoop/ozone/om/KeyManager.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 111 ++--
.../om/OMDBCheckpointServletInodeBasedXfer.java | 76 ++-
.../hadoop/ozone/om/OmSnapshotLocalData.java | 289 +++++++++++
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 259 ++++++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 81 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 78 ++-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +-
.../hadoop/ozone/om/PendingKeysDeletion.java | 16 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +
.../multitenant/MultiTenantAccessController.java | 36 ++
.../ozone/om/ratis/OzoneManagerRatisServer.java | 7 +
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 14 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 12 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 7 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +
.../om/request/key/acl/OMKeySetAclRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 38 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 6 +-
.../request/s3/tenant/OMTenantDeleteRequest.java | 21 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 15 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 13 +-
.../snapshot/OMSnapshotMoveTableKeysRequest.java | 13 +-
.../om/request/snapshot/OMSnapshotMoveUtils.java | 56 ++
.../S3MultipartUploadCommitPartResponse.java | 62 +--
...S3MultipartUploadCommitPartResponseWithFSO.java | 19 +-
.../ozone/om/service/DirectoryDeletingService.java | 65 ++-
.../ozone/om/service/KeyDeletingService.java | 136 ++++-
.../ozone/om/service/SnapshotDeletingService.java | 7 +-
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 28 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 1 +
.../ozone/om/snapshot/SnapshotDiffManager.java | 25 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 12 +-
.../OzoneDelegationTokenSecretManager.java | 4 +
.../ozone/security/acl/OzoneAuthorizerFactory.java | 44 +-
.../security/acl/OzoneManagerAuthorizer.java} | 16 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 44 +-
.../webapps/ozoneManager/om-overview.html | 61 ++-
.../resources/webapps/ozoneManager/ozoneManager.js | 43 +-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 18 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 23 +-
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 212 ++++++++
.../hadoop/ozone/om/TestOmSnapshotManager.java | 118 +++--
.../ozone/om/TestOmSnapshotManagerConfig.java | 67 +++
.../ozone/om/request/OMRequestTestUtils.java | 30 +-
.../TestOMDirectoriesPurgeRequestAndResponse.java | 6 +
.../om/request/key/TestOMKeyCommitRequest.java | 119 ++++-
.../om/request/key/TestOMKeyDeleteRequest.java | 2 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 32 ++
.../key/TestOMKeyPurgeRequestAndResponse.java | 5 +
.../s3/multipart/TestS3MultipartRequest.java | 40 +-
.../TestS3MultipartUploadCommitPartRequest.java | 365 +++++++++++++
...tS3MultipartUploadCommitPartRequestWithFSO.java | 19 +
.../s3/multipart/TestS3MultipartResponse.java | 15 +-
.../om/service/TestDirectoryDeletingService.java | 80 ++-
.../ozone/om/service/TestKeyDeletingService.java | 189 ++++++-
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +
.../ozone/security/TestOzoneTokenIdentifier.java | 11 +-
.../security/acl/TestOzoneAdministrators.java | 58 ++-
.../src/test/resources/junit-platform.properties | 27 +
.../recon/schema/ContainerSchemaDefinition.java | 8 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 77 ++-
.../ozone/recon/api/ClusterStateEndpoint.java | 3 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 50 +-
.../ozone/recon/api/OMDBInsightEndpoint.java | 31 +-
.../recon/api/types/KeyEntityInfoProtoWrapper.java | 144 ------
.../ozone/recon/api/types/ListKeysResponse.java | 6 +-
.../ozone/recon/api/types/ReconBasicOmKeyInfo.java | 278 ++++++++++
.../api/types/UnhealthyContainersResponse.java | 33 ++
.../persistence/ContainerHealthSchemaManager.java | 58 ++-
.../recon/recovery/ReconOMMetadataManager.java | 6 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 6 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 2 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 8 +
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 103 +++-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 41 +-
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 191 ++++---
.../recon/ozone-recon-web/src/utils/common.tsx | 2 +-
.../components/duBreadcrumbNav/duBreadcrumbNav.tsx | 2 +-
.../src/v2/components/navBar/navBar.tsx | 4 +-
.../duMetadata.tsx => nuMetadata/nuMetadata.tsx} | 4 +-
.../overviewCard/overviewStorageCard.tsx | 2 +-
.../plots/{duPieChart.tsx => nuPieChart.tsx} | 18 +-
.../namespaceUsage.less} | 0
.../namespaceUsage.tsx} | 47 +-
.../recon/ozone-recon-web/src/v2/routes-v2.tsx | 6 +-
...{diskUsage.types.ts => namespaceUsage.types.ts} | 6 +-
.../webapps/recon/ozone-recon-web/vite.config.ts | 3 +
.../ozone/recon/api/TestContainerEndpoint.java | 34 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 3 +-
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 18 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../TestNSSummaryTaskControllerIntegration.java | 570 +++++++++++++++++++++
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 7 +-
.../recon/tasks/TestNSSummaryUnifiedControl.java | 551 ++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 88 ++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 49 +-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 18 +
.../hadoop/ozone/repair/TransactionInfoRepair.java | 12 +-
.../ozone/repair/ldb/RocksDBManualCompaction.java | 12 +-
.../ozone/repair/om/SnapshotChainRepair.java | 11 +-
.../ozone/repair/TestTransactionInfoRepair.java | 9 +-
.../hadoop/ozone/repair/ldb/TestLdbRepair.java | 342 +++++++++++++
.../ozone/repair/om/TestSnapshotChainRepair.java | 19 +-
pom.xml | 49 +-
336 files changed, 13716 insertions(+), 2353 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index dcd525d09d1..adcc0b76d2f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -90,6 +90,10 @@ When creating a new jira for any kind of new feature, improvement or bug, please
* If it is follow up of another issue, please link the previous jira to it so that context is preserve.
3. **Jira examples:** [HDDS-9272](https://issues.apache.org/jira/browse/HDDS-9272), [HDDS-9322](https://issues.apache.org/jira/browse/HDDS-9322), [HDDS-9291](https://issues.apache.org/jira/browse/HDDS-9291), [HDDS-8940](https://issues.apache.org/jira/browse/HDDS-8940), [HDDS-9282](https://issues.apache.org/jira/browse/HDDS-9282)
+## New feature development
+
+For large feature development changes, we use a process called "Ozone Enhancement Proposals" (OEP). This process is designed to ensure that major changes to Ozone are well-designed and have community consensus. If you are planning to propose a significant change, please read the [Ozone Enhancement Proposals](https://ozone.apache.org/docs/edge/design/ozone-enhancement-proposals.html) documentation and create a design document before you start coding. Please note that we only accept design [...]
+
## Contribute your modifications
We use GitHub pull requests for contributing changes to the repository. The main workflow is as follows:
diff --git a/hadoop-ozone/dist/src/main/smoketest/kinit.robot b/dev-support/byteman/container-state-template.btm
similarity index 67%
copy from hadoop-ozone/dist/src/main/smoketest/kinit.robot
copy to dev-support/byteman/container-state-template.btm
index c9c1b754178..d1ca5449f5c 100644
--- a/hadoop-ozone/dist/src/main/smoketest/kinit.robot
+++ b/dev-support/byteman/container-state-template.btm
@@ -13,16 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-*** Settings ***
-Documentation Kinit test user
-Library OperatingSystem
-Resource commonlib.robot
-Test Timeout 2 minute
-
-
-*** Variables ***
-${testuser} testuser
-
-*** Test Cases ***
-Kinit
- Kinit test user ${testuser} ${testuser}.keytab
+RULE Change container state to STATE_PLACEHOLDER
+CLASS org.apache.hadoop.ozone.container.common.impl.ContainerData
+METHOD getState
+AT ENTRY
+IF TRUE
+DO
+ traceln("BYTEMAN RULE: Overriding getState() to return STATE_PLACEHOLDER state");
+ return org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos$ContainerDataProto$State.valueOf("STATE_PLACEHOLDER")
+ENDRULE
diff --git a/hadoop-ozone/dist/src/main/smoketest/kinit.robot b/dev-support/byteman/fail-create-bucket.btm
similarity index 68%
copy from hadoop-ozone/dist/src/main/smoketest/kinit.robot
copy to dev-support/byteman/fail-create-bucket.btm
index c9c1b754178..f624e45717c 100644
--- a/hadoop-ozone/dist/src/main/smoketest/kinit.robot
+++ b/dev-support/byteman/fail-create-bucket.btm
@@ -13,16 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-*** Settings ***
-Documentation Kinit test user
-Library OperatingSystem
-Resource commonlib.robot
-Test Timeout 2 minute
-
-
-*** Variables ***
-${testuser} testuser
+#
+# This script instruments ozone manager to fail a CreateBucket request for a specific name
+#
-*** Test Cases ***
-Kinit
- Kinit test user ${testuser} ${testuser}.keytab
+RULE Crash OM with CreateBucket
+CLASS org.apache.hadoop.ozone.om.request.bucket.OMBucketCreateRequest
+METHOD validateAndUpdateCache
+AT ENTRY
+IF TRUE
+DO
+ traceln("--> crashing CreateBucket request");
+ THROW new RuntimeException("Byteman crashes OM");
+ENDRULE
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index d9db5ade573..8a53bcf7aac 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -259,7 +259,7 @@ public final class HddsConfigKeys {
public static final String HDDS_SECRET_KEY_EXPIRY_DURATION =
"hdds.secret.key.expiry.duration";
- public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "7d";
+ public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "9d";
public static final String HDDS_SECRET_KEY_ROTATE_DURATION =
"hdds.secret.key.rotate.duration";
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
index dbab95efd4b..bc644247a2d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
@@ -104,7 +104,7 @@ public class ScmConfig extends ReconfigurableConfig {
@Config(key = "hdds.scm.block.deletion.per-interval.max",
type = ConfigType.INT,
- defaultValue = "100000",
+ defaultValue = "500000",
reconfigurable = true,
tags = { ConfigTag.SCM, ConfigTag.DELETION},
description =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 44229338ec6..e4cc9827b98 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -456,6 +456,11 @@ public final class ScmConfigKeys {
"ozone.scm.block.deletion.max.retry";
public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
+ public static final String OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR =
+ "ozone.scm.block.deletion.per.dn.distribution.factor";
+
+ public static final int OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR_DEFAULT = 8;
+
public static final String OZONE_SCM_SEQUENCE_ID_BATCH_SIZE =
"ozone.scm.sequence.id.batch.size";
public static final int OZONE_SCM_SEQUENCE_ID_BATCH_SIZE_DEFAULT = 1000;
@@ -521,7 +526,7 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE =
"ozone.scm.ha.ratis.segment.size";
- public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE_DEFAULT = "4MB";
+ public static final String OZONE_SCM_HA_RAFT_SEGMENT_SIZE_DEFAULT = "64MB";
public static final String OZONE_SCM_HA_RAFT_SEGMENT_PRE_ALLOCATED_SIZE =
"ozone.scm.ha.ratis.segment.preallocated.size";
@@ -617,7 +622,7 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL =
"ozone.scm.ha.dbtransactionbuffer.flush.interval";
public static final long
- OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT = 600 * 1000L;
+ OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT = 60 * 1000L;
public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
"net.topology.node.switch.mapping.impl";
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
index 02e68515f38..f6ac0a4872c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
@@ -41,7 +41,8 @@ public enum HDDSLayoutFeature implements LayoutFeature {
HADOOP_PRC_PORTS_IN_DATANODEDETAILS(7, "Adding Hadoop RPC ports " +
"to DatanodeDetails."),
HBASE_SUPPORT(8, "Datanode RocksDB Schema Version 3 has an extra table " +
- "for the last chunk of blocks to support HBase.)");
+ "for the last chunk of blocks to support HBase.)"),
+ WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value type as proto");
////////////////////////////// //////////////////////////////
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
index 7ddd20bde35..ca45e73ac1b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
@@ -77,11 +77,11 @@ protected CompletableFuture<Void> getFuture() {
}
@VisibleForTesting
- public ExecutorService getExecutorService() {
+ public synchronized ExecutorService getExecutorService() {
return this.exec;
}
- public void setPoolSize(int size) {
+ public synchronized void setPoolSize(int size) {
if (size <= 0) {
throw new IllegalArgumentException("Pool size must be positive.");
}
@@ -108,6 +108,7 @@ public void runPeriodicalTaskNow() throws Exception {
while (!tasks.isEmpty()) {
tasks.poll().call();
}
+ execTaskCompletion();
}
// start service
@@ -125,15 +126,30 @@ protected synchronized void setInterval(long newInterval, TimeUnit newUnit) {
this.unit = newUnit;
}
+ protected synchronized long getIntervalMillis() {
+ return this.unit.toMillis(interval);
+ }
+
public abstract BackgroundTaskQueue getTasks();
+ protected void execTaskCompletion() { }
+
/**
* Run one or more background tasks concurrently.
* Wait until all tasks to return the result.
*/
public class PeriodicalTask implements Runnable {
@Override
- public synchronized void run() {
+ public void run() {
+ // wait for previous set of tasks to complete
+ try {
+ future.join();
+ } catch (RuntimeException e) {
+ LOG.error("Background service execution failed.", e);
+ } finally {
+ execTaskCompletion();
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("Running background service : {}", serviceName);
}
@@ -146,35 +162,36 @@ public synchronized void run() {
if (LOG.isDebugEnabled()) {
LOG.debug("Number of background tasks to execute : {}", tasks.size());
}
-
- while (!tasks.isEmpty()) {
- BackgroundTask task = tasks.poll();
- future = future.thenCombine(CompletableFuture.runAsync(() -> {
- long startTime = System.nanoTime();
- try {
- BackgroundTaskResult result = task.call();
- if (LOG.isDebugEnabled()) {
- LOG.debug("task execution result size {}", result.getSize());
- }
- } catch (Throwable e) {
- LOG.error("Background task execution failed", e);
- if (e instanceof Error) {
- throw (Error) e;
- }
- } finally {
- long endTime = System.nanoTime();
- if (endTime - startTime > serviceTimeoutInNanos) {
- LOG.warn("{} Background task execution took {}ns > {}ns(timeout)",
- serviceName, endTime - startTime, serviceTimeoutInNanos);
+ synchronized (BackgroundService.this) {
+ while (!tasks.isEmpty()) {
+ BackgroundTask task = tasks.poll();
+ future = future.thenCombine(CompletableFuture.runAsync(() -> {
+ long startTime = System.nanoTime();
+ try {
+ BackgroundTaskResult result = task.call();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("task execution result size {}", result.getSize());
+ }
+ } catch (Throwable e) {
+ LOG.error("Background task execution failed", e);
+ if (e instanceof Error) {
+ throw (Error) e;
+ }
+ } finally {
+ long endTime = System.nanoTime();
+ if (endTime - startTime > serviceTimeoutInNanos) {
+ LOG.warn("{} Background task execution took {}ns > {}ns(timeout)",
+ serviceName, endTime - startTime, serviceTimeoutInNanos);
+ }
}
- }
- }, exec), (Void1, Void) -> null);
+ }, exec).exceptionally(e -> null), (Void1, Void) -> null);
+ }
}
}
}
// shutdown and make sure all threads are properly released.
- public void shutdown() {
+ public synchronized void shutdown() {
LOG.info("Shutting down service {}", this.serviceName);
exec.shutdown();
try {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SlidingWindow.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SlidingWindow.java
new file mode 100644
index 00000000000..744e842a398
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SlidingWindow.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * A sliding window implementation that combines time-based expiry with a
+ * maximum size constraint. The window tracks event timestamps and maintains two
+ * limits:
+ * <ul>
+ * <li>Time-based: Events older than the specified expiry duration are
+ * automatically removed
+ * <li>Size-based: The window maintains at most windowSize latest events, removing
+ * older events when this limit is exceeded
+ * </ul>
+ *
+ * The window is considered full when the number of non-expired events exceeds
+ * the specified window size. Events are automatically pruned based on both
+ * their age and the maximum size constraint.
+ */
+public class SlidingWindow {
+ private final Object lock = new Object();
+ private final int windowSize;
+ private final Deque<Long> timestamps;
+ private final long expiryDurationMillis;
+ private final Clock clock;
+
+ /**
+ * Default constructor that uses a monotonic clock.
+ *
+ * @param windowSize the maximum number of events that are tracked
+ * @param expiryDuration the duration after which an entry in the window expires
+ */
+ public SlidingWindow(int windowSize, Duration expiryDuration) {
+ this(windowSize, expiryDuration, new MonotonicClock());
+ }
+
+ /**
+ * Constructor with a custom clock for testing.
+ *
+ * @param windowSize the maximum number of events that are tracked
+ * @param expiryDuration the duration after which an entry in the window expires
+ * @param clock the clock to use for time measurements
+ */
+ public SlidingWindow(int windowSize, Duration expiryDuration, Clock clock) {
+ if (windowSize < 0) {
+ throw new IllegalArgumentException("Window size must be greater than 0");
+ }
+ if (expiryDuration.isNegative() || expiryDuration.isZero()) {
+ throw new IllegalArgumentException("Expiry duration must be greater than 0");
+ }
+ this.windowSize = windowSize;
+ this.expiryDurationMillis = expiryDuration.toMillis();
+ this.clock = clock;
+ // We limit the initial queue size to 100 to control the memory usage
+ this.timestamps = new ArrayDeque<>(Math.min(windowSize + 1, 100));
+ }
+
+ public void add() {
+ synchronized (lock) {
+ if (isExceeded()) {
+ timestamps.remove();
+ }
+
+ timestamps.add(getCurrentTime());
+ }
+ }
+
+ /**
+ * Checks if the sliding window has exceeded its maximum size.
+ * This is useful to track if we have encountered more events than the window's defined limit.
+ * @return true if the number of tracked timestamps in the sliding window
+ * exceeds the specified window size, false otherwise.
+ */
+ public boolean isExceeded() {
+ synchronized (lock) {
+ removeExpired();
+ return timestamps.size() > windowSize;
+ }
+ }
+
+ /**
+ * Returns the current number of events that are tracked within the sliding window queue.
+ * The number of events can exceed the window size.
+ * This method ensures that expired events are removed before computing the count.
+ *
+ * @return the number of valid timestamps currently in the sliding window
+ */
+ @VisibleForTesting
+ public int getNumEvents() {
+ synchronized (lock) {
+ removeExpired();
+ return timestamps.size();
+ }
+ }
+
+ /**
+ * Returns the current number of events that are tracked within the sliding window queue.
+ * The number of events cannot exceed the window size.
+ * This method ensures that expired events are removed before computing the count.
+ *
+ * @return the number of valid timestamps currently in the sliding window
+ */
+ public int getNumEventsInWindow() {
+ synchronized (lock) {
+ removeExpired();
+ return Math.min(timestamps.size(), windowSize);
+ }
+ }
+
+ private void removeExpired() {
+ synchronized (lock) {
+ long currentTime = getCurrentTime();
+ long expirationThreshold = currentTime - expiryDurationMillis;
+
+ while (!timestamps.isEmpty() && timestamps.peek() < expirationThreshold) {
+ timestamps.remove();
+ }
+ }
+ }
+
+ public int getWindowSize() {
+ return windowSize;
+ }
+
+ private long getCurrentTime() {
+ return clock.millis();
+ }
+
+ /**
+ * A custom monotonic clock implementation.
+ * Implementation of Clock that uses System.nanoTime() for real usage.
+ * See {@see org.apache.ozone.test.TestClock}
+ */
+ private static final class MonotonicClock extends Clock {
+ @Override
+ public long millis() {
+ return TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+ }
+
+ @Override
+ public java.time.Instant instant() {
+ return java.time.Instant.ofEpochMilli(millis());
+ }
+
+ @Override
+ public java.time.ZoneId getZone() {
+ return java.time.ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(java.time.ZoneId zone) {
+ // Ignore zone for monotonic clock
+ throw new UnsupportedOperationException("Sliding Window class does not allow changing the timezone");
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
index 272e862d680..af0772769a1 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
@@ -114,6 +114,17 @@ public T copyObject(T message) {
}
}
+ public static <T, DELEGATE> DelegatedCodec<T, DELEGATE> decodeOnly(
+ Codec<DELEGATE> delegate, CheckedFunction<DELEGATE, T, CodecException> forward, Class<T> clazz) {
+ return new DelegatedCodec<>(delegate, forward, unsupportedBackward(), clazz, CopyType.DEEP);
+ }
+
+ private static <A, B> CheckedFunction<A, B, CodecException> unsupportedBackward() {
+ return a -> {
+ throw new UnsupportedOperationException("Unsupported backward conversion");
+ };
+ }
+
@Override
public String toString() {
return name;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 7df44d721dd..0008e48f354 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -88,6 +88,7 @@ public final class OzoneConsts {
public static final Path ROOT_PATH = Paths.get(OZONE_ROOT);
public static final String CONTAINER_EXTENSION = ".container";
+ public static final String CONTAINER_DATA_CHECKSUM_EXTENSION = ".tree";
public static final String CONTAINER_META_PATH = "metadata";
public static final String CONTAINER_TEMPORARY_CHUNK_PREFIX = "tmp";
public static final String CONTAINER_CHUNK_NAME_DELIMITER = ".";
@@ -141,6 +142,7 @@ public final class OzoneConsts {
public static final String CONTAINER_BYTES_USED = "#BYTESUSED";
public static final String PENDING_DELETE_BLOCK_COUNT =
"#PENDINGDELETEBLOCKCOUNT";
+ public static final String CONTAINER_DATA_CHECKSUM = "#DATACHECKSUM";
/**
* OM LevelDB prefixes.
@@ -205,6 +207,15 @@ public final class OzoneConsts {
public static final String SCM_CONTEXT_ATTRIBUTE = "ozone.scm";
+ // YAML field constants for OmSnapshotLocalData (thus the OM_SLD_ prefix) YAML files
+ public static final String OM_SLD_VERSION = "version";
+ public static final String OM_SLD_CHECKSUM = "checksum";
+ public static final String OM_SLD_IS_SST_FILTERED = "isSSTFiltered";
+ public static final String OM_SLD_UNCOMPACTED_SST_FILE_LIST = "uncompactedSSTFileList";
+ public static final String OM_SLD_LAST_COMPACTION_TIME = "lastCompactionTime";
+ public static final String OM_SLD_NEEDS_COMPACTION = "needsCompaction";
+ public static final String OM_SLD_COMPACTED_SST_FILE_LIST = "compactedSSTFileList";
+
// YAML fields for .container files
public static final String CONTAINER_ID = "containerID";
public static final String CONTAINER_TYPE = "containerType";
@@ -536,6 +547,12 @@ public final class OzoneConsts {
*/
public static final String ETAG = "ETag";
+ /**
+ * A constant string used as a separator in various contexts within
+ * the OMDBCheckpoint functions.
+ */
+ public static final String OM_SNAPSHOT_SEPARATOR = "-";
+
private OzoneConsts() {
// Never Constructed
}
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 9e3f5fa8b32..b015e76d3d5 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -321,8 +321,8 @@
<name>hdds.container.ratis.segment.size</name>
<value>64MB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the raft segment file used
- by Apache Ratis on datanodes. (64 MB by default)
+ <description>The size of the raft segment file used by Apache Ratis
+ on datanodes. (64 MB by default)
</description>
</property>
<property>
@@ -484,7 +484,7 @@
</property>
<property>
<name>ozone.key.deleting.limit.per.task</name>
- <value>20000</value>
+ <value>50000</value>
<tag>OM, PERFORMANCE</tag>
<description>
A maximum number of keys to be scanned by key deleting service
@@ -795,6 +795,16 @@
times SCM is going to retry sending a deletion operation to the data node.
</description>
</property>
+ <property>
+ <name>ozone.scm.block.deletion.per.dn.distribution.factor</name>
+ <value>8</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ Factor with which number of delete blocks sent to each datanode in every interval.
+ If total number of DNs are 100 and hdds.scm.block.deletion.per-interval.max is 500000
+ Then maximum 500000/(100/8) = 40000 blocks will be sent to each DN in every interval.
+ </description>
+ </property>
<property>
<name>ozone.scm.block.size</name>
<value>256MB</value>
@@ -2170,10 +2180,10 @@
<property>
<name>ozone.om.ratis.segment.size</name>
- <value>4MB</value>
+ <value>64MB</value>
<tag>OZONE, OM, RATIS, PERFORMANCE</tag>
<description>The size of the raft segment used by Apache Ratis on OM.
- (4 MB by default)
+ (64 MB by default)
</description>
</property>
@@ -2182,7 +2192,7 @@
<value>4MB</value>
<tag>OZONE, OM, RATIS, PERFORMANCE</tag>
<description>The size of the buffer which is preallocated for raft segment
- used by Apache Ratis on OM.(4 MB by default)
+ used by Apache Ratis on OM. (4 MB by default)
</description>
</property>
@@ -2200,6 +2210,20 @@
<description>Byte limit for Raft's Log Worker queue.
</description>
</property>
+ <property>
+ <name>ozone.om.ratis.server.pending.write.byte-limit</name>
+ <value>64MB</value>
+ <tag>OZONE, DEBUG, OM, RATIS</tag>
+ <description>Maximum byte size of all pending write requests.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.ratis.server.pending.write.element-limit</name>
+ <value>4096</value>
+ <tag>OZONE, DEBUG, OM, RATIS</tag>
+ <description>Maximum number of pending write requests.
+ </description>
+ </property>
<property>
<name>ozone.om.ratis.log.purge.gap</name>
<value>1000000</value>
@@ -2790,8 +2814,14 @@
<name>ozone.manager.delegation.token.max-lifetime</name>
<value>7d</value>
<description>
- Default max time interval after which ozone delegation token will
- not be renewed.
+ Default max time interval after which ozone delegation token will not be renewed.
+ Delegation Token is signed and verified using secret key which has a max hdds.secret.key.expiry.duration lifetime.
+ To guarantee that the delegation token can be properly loaded, verified, and renewed during its lifetime,
+ (ozone.manager.delegation.token.max-lifetime + hdds.secret.key.rotate.duration + ozone.manager.delegation.remover.scan.interval)
+ must not be greater than hdds.secret.key.expiry.duration.
+ If any of ozone.manager.delegation.token.max-lifetime, hdds.secret.key.expiry.duration, hdds.secret.key.rotate.duration
+ or ozone.manager.delegation.remover.scan.interval value is changed, The above constrain must be checked and
+ values be adjusted accordingly if necessary.
</description>
</property>
@@ -3853,10 +3883,10 @@
</property>
<property>
<name>ozone.scm.ha.ratis.segment.size</name>
- <value>4MB</value>
+ <value>64MB</value>
<tag>SCM, OZONE, HA, RATIS</tag>
<description>The size of the raft segment used by Apache Ratis on
- SCM. (4 MB by default)
+ SCM. (64 MB by default)
</description>
</property>
<property>
@@ -3864,7 +3894,7 @@
<value>4MB</value>
<tag>SCM, OZONE, HA, RATIS</tag>
<description>The size of the buffer which is preallocated for
- raft segment used by Apache Ratis on SCM.(4 MB by default)
+ raft segment used by Apache Ratis on SCM. (4 MB by default)
</description>
</property>
<property>
@@ -3993,7 +4023,7 @@
</property>
<property>
<name>ozone.scm.ha.dbtransactionbuffer.flush.interval</name>
- <value>600s</value>
+ <value>60s</value>
<tag>SCM, OZONE</tag>
<description>Wait duration for flush of buffered transaction.</description>
</property>
@@ -4575,12 +4605,18 @@
</property>
<property>
<name>hdds.secret.key.expiry.duration</name>
- <value>7d</value>
+ <value>9d</value>
<tag>SCM, SECURITY</tag>
<description>
The duration for which symmetric secret keys issued by SCM are valid.
- This default value, in combination with hdds.secret.key.rotate.duration=1d, results in 7 secret keys (for the
- last 7 days) are kept valid at any point of time.
+ Secret key is used to sign delegation tokens signed by OM, so the secret key must be valid for at least
+ (ozone.manager.delegation.token.max-lifetime + hdds.secret.key.rotate.duration + ozone.manager.delegation.remover.scan.interval)
+ time to guarantee that delegation tokens can be verified by OM. Considering the default value of three properties
+ mentioned and rounding up to days, this property's default value, in combination with hdds.secret.key.rotate.duration=1d,
+ results in 9 secret keys (for the last 9 days) are kept valid at any point of time.
+ If any of ozone.manager.delegation.token.max-lifetime, hdds.secret.key.rotate.duration or
+ ozone.manager.delegation.remover.scan.interval value is changed, this property should be checked, and updated
+ accordingly if necessary.
</description>
</property>
<property>
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java
new file mode 100644
index 00000000000..369426bcfd0
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSlidingWindow.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import org.apache.ozone.test.TestClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link SlidingWindow} class.
+ */
+class TestSlidingWindow {
+
+ private TestClock testClock;
+
+ @BeforeEach
+ void setup() {
+ testClock = TestClock.newInstance();
+ }
+
+ @Test
+ void testConstructorValidation() {
+ // Test invalid window size
+ assertThrows(IllegalArgumentException.class, () -> new SlidingWindow(-1, Duration.ofMillis(100)));
+
+ // Test invalid expiry duration
+ assertThrows(IllegalArgumentException.class, () -> new SlidingWindow(1, Duration.ofMillis(0)));
+ assertThrows(IllegalArgumentException.class, () -> new SlidingWindow(1, Duration.ofMillis(-1)));
+ }
+
+ @Test
+ void testAdd() {
+ SlidingWindow slidingWindow = new SlidingWindow(3, Duration.ofSeconds(5), testClock);
+ for (int i = 0; i < slidingWindow.getWindowSize(); i++) {
+ slidingWindow.add();
+ assertEquals(i + 1, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+ }
+
+ slidingWindow.add();
+ assertEquals(slidingWindow.getWindowSize() + 1, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+ }
+
+ @Test
+ void testEventExpiration() {
+ SlidingWindow slidingWindow = new SlidingWindow(2, Duration.ofMillis(500), testClock);
+
+ // Add events to reach threshold
+ slidingWindow.add();
+ slidingWindow.add();
+ slidingWindow.add();
+ assertEquals(3, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+
+ // Fast forward time to expire events
+ testClock.fastForward(600);
+
+ assertEquals(0, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+
+ // Add one more event - should not be enough to mark as full
+ slidingWindow.add();
+ assertEquals(1, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+ }
+
+ @Test
+ void testPartialExpiration() {
+ SlidingWindow slidingWindow = new SlidingWindow(3, Duration.ofSeconds(1), testClock);
+
+ slidingWindow.add();
+ slidingWindow.add();
+ slidingWindow.add();
+ slidingWindow.add();
+ assertEquals(4, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+
+ testClock.fastForward(600);
+ slidingWindow.add(); // this will remove the oldest event as the window is full
+ assertEquals(4, slidingWindow.getNumEvents());
+
+ // Fast forward time to expire the oldest events
+ testClock.fastForward(500);
+ assertEquals(1, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+ }
+
+ @Test
+ void testZeroWindowSize() {
+ SlidingWindow slidingWindow = new SlidingWindow(0, Duration.ofSeconds(5), testClock);
+
+ // Verify initial state
+ assertEquals(0, slidingWindow.getWindowSize());
+ assertEquals(0, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+
+ // Add an event - with window size 0, any event should cause isExceeded to return true
+ slidingWindow.add();
+ assertEquals(1, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+
+ // Add another event - should replace the previous one as window is exceeded
+ slidingWindow.add();
+ assertEquals(1, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+
+ // Test expiration
+ testClock.fastForward(6000); // Move past expiry time
+ assertEquals(0, slidingWindow.getNumEvents());
+ assertFalse(slidingWindow.isExceeded());
+
+ // Add multiple events in sequence - should always keep only the latest one
+ for (int i = 0; i < 5; i++) {
+ slidingWindow.add();
+ assertEquals(1, slidingWindow.getNumEvents());
+ assertTrue(slidingWindow.isExceeded());
+ }
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/util/TestBackgroundService.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/util/TestBackgroundService.java
new file mode 100644
index 00000000000..2c5ed1bfea5
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/util/TestBackgroundService.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+/**
+ * Test class for validating the functionality of the base class BackgroundService.
+ */
+@Timeout(10)
+public class TestBackgroundService {
+
+ private BackgroundTaskQueue queue;
+ private BackgroundService backgroundService;
+ private AtomicInteger runCount;
+
+ private static class TestTask implements BackgroundTask {
+ private Map<Integer, Integer> map;
+ private Map<Integer, Lock> locks;
+ private int index;
+
+ TestTask(int index, Map<Integer, Integer> map, Map<Integer, Lock> locks) {
+ this.index = index;
+ this.map = map;
+ this.locks = locks;
+ locks.put(index, new ReentrantLock());
+ map.put(index, 0);
+ }
+
+ @Override
+ public BackgroundTaskResult call() {
+ locks.get(index).lock();
+ try {
+ map.merge(index, 1, Integer::sum);
+ } finally {
+ locks.get(index).unlock();
+ }
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
+ }
+ }
+
+ @BeforeEach
+ public void setUp() {
+ queue = new BackgroundTaskQueue();
+ runCount = new AtomicInteger(-1);
+ }
+
+ @AfterEach
+ public void shutDown() {
+ if (backgroundService != null) {
+ backgroundService.shutdown();
+ }
+ }
+
+ private BackgroundService createBackgroundService(String name, int interval,
+ TimeUnit unit, int workerThreads, int serviceTimeout) {
+ return new BackgroundService(name, interval, unit, workerThreads, serviceTimeout) {
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ return queue;
+ }
+
+ @Override
+ public void execTaskCompletion() {
+ runCount.incrementAndGet();
+ }
+ };
+ }
+
+ @Test
+ public void testBackgroundServiceRunWaitsForTasks() throws InterruptedException, TimeoutException {
+ int interval = 100;
+ int serviceTimeout = 3000;
+ int nTasks = 10;
+ List<Integer> expValuesWithEvenLocks = Arrays.asList(0, 1, 0, 1, 0, 1, 0, 1, 0, 1);
+ List<Integer> expFinalValues = Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+ Map<Integer, Integer> map = new HashMap<>();
+ Map<Integer, Lock> locks = new HashMap<>();
+ // Add tasks to the queue.
+ IntStream.range(0, nTasks).forEach(i -> queue.add(new TestTask(i, map, locks)));
+ // Acquire locks on all even indices.
+ List<Lock> lockList = new ArrayList<>();
+ IntStream.range(0, nTasks).filter(i -> (i & 1) == 0).forEach(i -> {
+ lockList.add(locks.get(i));
+ locks.get(i).lock();
+ });
+ // Start the background service.
+ backgroundService = createBackgroundService("testWaitForTasks", interval, TimeUnit.MILLISECONDS,
+ nTasks, serviceTimeout);
+ backgroundService.start();
+ // Wait for odd index tasks to complete.
+ GenericTestUtils.waitFor(() ->
+ IntStream.range(1, nTasks).filter(i -> (i & 1) == 1).allMatch(i -> map.get(i) == 1),
+ interval / 5, serviceTimeout);
+ Thread.sleep(serviceTimeout);
+ // Verify that even index tasks have not run even after serviceTimeout.
+ assertEquals(expValuesWithEvenLocks, IntStream.range(0, nTasks).boxed().map(map::get).collect(Collectors.toList()));
+ // Background service is still waiting for all tasks to complete.
+ assertEquals(0, runCount.get());
+ // Release the locks on even index tasks.
+ lockList.forEach(Lock::unlock);
+ // Wait for current run of BackgroundService to complete.
+ GenericTestUtils.waitFor(() -> runCount.get() == 1, interval / 5, serviceTimeout);
+ // Verify that all tasks have completed.
+ assertEquals(expFinalValues, IntStream.range(0, nTasks).boxed().map(map::get).collect(Collectors.toList()));
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testSingleWorkerThread() throws InterruptedException, TimeoutException {
+ int nTasks = 5;
+ int interval = 100;
+ int serviceTimeout = 3000;
+ Map<Integer, Integer> map = new HashMap<>();
+ Map<Integer, Lock> locks = new HashMap<>();
+ List<Integer> expFinalValues = Arrays.asList(1, 1, 1, 1, 1);
+ // Add tasks to the queue.
+ IntStream.range(0, nTasks).forEach(i -> queue.add(new TestTask(i, map, locks)));
+ // Start the background service with a single worker thread.
+ backgroundService = createBackgroundService("testSingleWorkerThread", interval, TimeUnit.MILLISECONDS,
+ 1, serviceTimeout);
+ backgroundService.start();
+ // Wait till current run of BackgroundService completes.
+ GenericTestUtils.waitFor(() -> runCount.get() == 1, interval / 5, serviceTimeout);
+ // Verify that all tasks have completed.
+ assertEquals(expFinalValues, IntStream.range(0, nTasks).boxed().map(map::get).collect(Collectors.toList()));
+ assertTrue(queue.isEmpty());
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
index b52d0667c78..ce124754f00 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
@@ -19,6 +19,7 @@
import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static org.apache.hadoop.hdds.HddsUtils.checksumToString;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM_EXTENSION;
import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
import com.google.common.annotations.VisibleForTesting;
@@ -29,17 +30,16 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -47,7 +47,6 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.utils.SimpleStriped;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
@@ -94,15 +93,7 @@ public ContainerProtos.ContainerChecksumInfo writeContainerDataTree(ContainerDat
Lock writeLock = getLock(containerID);
writeLock.lock();
try {
- ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = null;
- try {
- // If the file is not present, we will create the data for the first time. This happens under a write lock.
- checksumInfoBuilder = readBuilder(data).orElse(ContainerProtos.ContainerChecksumInfo.newBuilder());
- } catch (IOException ex) {
- LOG.error("Failed to read container checksum tree file for container {}. Creating a new instance.",
- containerID, ex);
- checksumInfoBuilder = ContainerProtos.ContainerChecksumInfo.newBuilder();
- }
+ ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = readOrCreate(data).toBuilder();
ContainerProtos.ContainerMerkleTree treeProto = captureLatencyNs(metrics.getCreateMerkleTreeLatencyNS(),
tree::toProto);
@@ -129,16 +120,7 @@ public void markBlocksAsDeleted(KeyValueContainerData data, Collection<Long> del
Lock writeLock = getLock(containerID);
writeLock.lock();
try {
- ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = null;
- try {
- // If the file is not present, we will create the data for the first time. This happens under a write lock.
- checksumInfoBuilder = readBuilder(data)
- .orElse(ContainerProtos.ContainerChecksumInfo.newBuilder());
- } catch (IOException ex) {
- LOG.error("Failed to read container checksum tree file for container {}. Overwriting it with a new instance.",
- data.getContainerID(), ex);
- checksumInfoBuilder = ContainerProtos.ContainerChecksumInfo.newBuilder();
- }
+ ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = readOrCreate(data).toBuilder();
// Although the persisted block list should already be sorted, we will sort it here to make sure.
// This will automatically fix any bugs in the persisted order that may show up.
@@ -179,7 +161,7 @@ public ContainerDiffReport diff(ContainerProtos.ContainerChecksumInfo thisChecks
ContainerProtos.ContainerChecksumInfo peerChecksumInfo) throws
StorageContainerException {
- ContainerDiffReport report = new ContainerDiffReport();
+ ContainerDiffReport report = new ContainerDiffReport(thisChecksumInfo.getContainerID());
try {
captureLatencyNs(metrics.getMerkleTreeDiffLatencyNS(), () -> {
Preconditions.assertNotNull(thisChecksumInfo, "Datanode's checksum info is null.");
@@ -280,6 +262,8 @@ private void compareBlockMerkleTree(ContainerProtos.BlockMerkleTree thisBlockMer
List<ContainerProtos.ChunkMerkleTree> thisChunkMerkleTreeList = thisBlockMerkleTree.getChunkMerkleTreeList();
List<ContainerProtos.ChunkMerkleTree> peerChunkMerkleTreeList = peerBlockMerkleTree.getChunkMerkleTreeList();
int thisIdx = 0, peerIdx = 0;
+ long containerID = report.getContainerID();
+ long blockID = thisBlockMerkleTree.getBlockID();
// Step 1: Process both lists while elements are present in both
while (thisIdx < thisChunkMerkleTreeList.size() && peerIdx < peerChunkMerkleTreeList.size()) {
@@ -293,8 +277,8 @@ private void compareBlockMerkleTree(ContainerProtos.BlockMerkleTree thisBlockMer
// thisTree = Healthy, peerTree = unhealthy -> Do nothing as thisTree is healthy.
// thisTree = Unhealthy, peerTree = Unhealthy -> Do Nothing as both are corrupt.
if (thisChunkMerkleTree.getDataChecksum() != peerChunkMerkleTree.getDataChecksum() &&
- !thisChunkMerkleTree.getChecksumMatches() && peerChunkMerkleTree.getChecksumMatches()) {
- report.addCorruptChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTree);
+ !thisChunkMerkleTree.getChecksumMatches()) {
+ reportChunkIfHealthy(containerID, blockID, peerChunkMerkleTree, report::addCorruptChunk);
}
thisIdx++;
peerIdx++;
@@ -304,14 +288,14 @@ private void compareBlockMerkleTree(ContainerProtos.BlockMerkleTree thisBlockMer
thisIdx++;
} else {
// Peer chunk's offset is smaller; record missing chunk and advance peerIdx
- report.addMissingChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTree);
+ reportChunkIfHealthy(containerID, blockID, peerChunkMerkleTree, report::addMissingChunk);
peerIdx++;
}
}
// Step 2: Process remaining chunks in the peer list
while (peerIdx < peerChunkMerkleTreeList.size()) {
- report.addMissingChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTreeList.get(peerIdx));
+ reportChunkIfHealthy(containerID, blockID, peerChunkMerkleTreeList.get(peerIdx), report::addMissingChunk);
peerIdx++;
}
@@ -319,28 +303,30 @@ private void compareBlockMerkleTree(ContainerProtos.BlockMerkleTree thisBlockMer
// chunks from us when they reconcile.
}
- public static long getDataChecksum(ContainerProtos.ContainerChecksumInfo checksumInfo) {
- return checksumInfo.getContainerMerkleTree().getDataChecksum();
+ private void reportChunkIfHealthy(long containerID, long blockID, ContainerProtos.ChunkMerkleTree peerTree,
+ BiConsumer<Long, ContainerProtos.ChunkMerkleTree> addToReport) {
+ if (peerTree.getChecksumMatches()) {
+ addToReport.accept(blockID, peerTree);
+ } else {
+ LOG.warn("Skipping chunk at offset {} in block {} of container {} since peer reported it as " +
+ "unhealthy.", peerTree.getOffset(), blockID, containerID);
+ }
}
- /**
- * Returns whether the container checksum tree file for the specified container exists without deserializing it.
- */
- public static boolean hasContainerChecksumFile(ContainerData data) {
- return getContainerChecksumFile(data).exists();
+ public static long getDataChecksum(ContainerProtos.ContainerChecksumInfo checksumInfo) {
+ return checksumInfo.getContainerMerkleTree().getDataChecksum();
}
/**
* Returns the container checksum tree file for the specified container without deserializing it.
*/
- @VisibleForTesting
public static File getContainerChecksumFile(ContainerData data) {
- return new File(data.getMetadataPath(), data.getContainerID() + ".tree");
+ return new File(data.getMetadataPath(), data.getContainerID() + CONTAINER_DATA_CHECKSUM_EXTENSION);
}
@VisibleForTesting
public static File getTmpContainerChecksumFile(ContainerData data) {
- return new File(data.getMetadataPath(), data.getContainerID() + ".tree.tmp");
+ return new File(data.getMetadataPath(), data.getContainerID() + CONTAINER_DATA_CHECKSUM_EXTENSION + ".tmp");
}
private Lock getLock(long containerID) {
@@ -348,21 +334,33 @@ private Lock getLock(long containerID) {
}
/**
+ * Reads the checksum info of the specified container. If the tree file with the information does not exist, an empty
+ * instance is returned.
* Callers are not required to hold a lock while calling this since writes are done to a tmp file and atomically
* swapped into place.
*/
- public Optional<ContainerProtos.ContainerChecksumInfo> read(ContainerData data) throws IOException {
+ public ContainerProtos.ContainerChecksumInfo read(ContainerData data) throws IOException {
try {
return captureLatencyNs(metrics.getReadContainerMerkleTreeLatencyNS(), () -> readChecksumInfo(data));
} catch (IOException ex) {
metrics.incrementMerkleTreeReadFailures();
- throw new IOException(ex);
+ throw ex;
}
}
- private Optional<ContainerProtos.ContainerChecksumInfo.Builder> readBuilder(ContainerData data) throws IOException {
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = read(data);
- return checksumInfo.map(ContainerProtos.ContainerChecksumInfo::toBuilder);
+ /**
+ * Reads the checksum info of the specified container. If the tree file with the information does not exist, or there
+ * is an exception trying to read the file, an empty instance is returned.
+ */
+ private ContainerProtos.ContainerChecksumInfo readOrCreate(ContainerData data) {
+ try {
+ // If the file is not present, we will create the data for the first time. This happens under a write lock.
+ return read(data);
+ } catch (IOException ex) {
+ LOG.error("Failed to read container checksum tree file for container {}. Overwriting it with a new instance.",
+ data.getContainerID(), ex);
+ return ContainerProtos.ContainerChecksumInfo.newBuilder().build();
+ }
}
/**
@@ -408,7 +406,7 @@ private SortedSet<Long> getDeletedBlockIDs(ContainerProtos.ContainerChecksumInfo
public ByteString getContainerChecksumInfo(KeyValueContainerData data) throws IOException {
File checksumFile = getContainerChecksumFile(data);
if (!checksumFile.exists()) {
- throw new NoSuchFileException("Checksum file does not exist for container #" + data.getContainerID());
+ throw new FileNotFoundException("Checksum file does not exist for container #" + data.getContainerID());
}
try (InputStream inStream = Files.newInputStream(checksumFile.toPath())) {
@@ -421,17 +419,18 @@ public ByteString getContainerChecksumInfo(KeyValueContainerData data) throws IO
* Callers are not required to hold a lock while calling this since writes are done to a tmp file and atomically
* swapped into place.
*/
- public static Optional<ContainerProtos.ContainerChecksumInfo> readChecksumInfo(ContainerData data)
+ public static ContainerProtos.ContainerChecksumInfo readChecksumInfo(ContainerData data)
throws IOException {
long containerID = data.getContainerID();
File checksumFile = getContainerChecksumFile(data);
try {
if (!checksumFile.exists()) {
LOG.debug("No checksum file currently exists for container {} at the path {}", containerID, checksumFile);
- return Optional.empty();
+ return ContainerProtos.ContainerChecksumInfo.newBuilder().build();
}
+
try (InputStream inStream = Files.newInputStream(checksumFile.toPath())) {
- return Optional.of(ContainerProtos.ContainerChecksumInfo.parseFrom(inStream));
+ return ContainerProtos.ContainerChecksumInfo.parseFrom(inStream);
}
} catch (IOException ex) {
throw new IOException("Error occurred when reading container merkle tree for containerID "
@@ -443,10 +442,4 @@ public static Optional<ContainerProtos.ContainerChecksumInfo> readChecksumInfo(C
public ContainerMerkleTreeMetrics getMetrics() {
return this.metrics;
}
-
- public static boolean checksumFileExist(Container<?> container) {
- File checksumFile = getContainerChecksumFile(container.getContainerData());
- return checksumFile.exists();
- }
-
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerDiffReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerDiffReport.java
index 6de3057b00c..882c9a7e831 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerDiffReport.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerDiffReport.java
@@ -31,11 +31,17 @@ public class ContainerDiffReport {
private final List<ContainerProtos.BlockMerkleTree> missingBlocks;
private final Map<Long, List<ContainerProtos.ChunkMerkleTree>> missingChunks;
private final Map<Long, List<ContainerProtos.ChunkMerkleTree>> corruptChunks;
+ private final long containerID;
- public ContainerDiffReport() {
+ public ContainerDiffReport(long containerID) {
this.missingBlocks = new ArrayList<>();
this.missingChunks = new HashMap<>();
this.corruptChunks = new HashMap<>();
+ this.containerID = containerID;
+ }
+
+ public long getContainerID() {
+ return containerID;
}
/**
@@ -105,7 +111,7 @@ public long getNumMissingBlocks() {
@Override
public String toString() {
- return "ContainerDiffReport:" +
+ return "Diff report for container " + containerID + ":" +
" Missing Blocks: " + getNumMissingBlocks() +
" Missing Chunks: " + getNumMissingChunks() + " chunks from " + missingChunks.size() + " blocks" +
" Corrupt Chunks: " + getNumCorruptChunks() + " chunks from " + corruptChunks.size() + " blocks";
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
index 3fc1f4fb951..69921885915 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
@@ -172,7 +172,10 @@ public void addChunks(ChunkMerkleTreeWriter... chunks) {
public ContainerProtos.BlockMerkleTree toProto() {
ContainerProtos.BlockMerkleTree.Builder blockTreeBuilder = ContainerProtos.BlockMerkleTree.newBuilder();
ChecksumByteBuffer checksumImpl = CHECKSUM_BUFFER_SUPPLIER.get();
- ByteBuffer blockChecksumBuffer = ByteBuffer.allocate(Long.BYTES * offset2Chunk.size());
+ // Allocate space for block ID + all chunk checksums
+ ByteBuffer blockChecksumBuffer = ByteBuffer.allocate(Long.BYTES * (1 + offset2Chunk.size()));
+ // Hash the block ID into the beginning of the block checksum calculation
+ blockChecksumBuffer.putLong(blockID);
for (ChunkMerkleTreeWriter chunkTree: offset2Chunk.values()) {
// Ordering of checksums within a chunk is assumed to be in the order they are written.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index 11092fe1f7b..f79c7e3f1df 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -99,6 +99,7 @@ public abstract class ContainerData {
// Checksum of the data within the container.
private long dataChecksum;
+ private static final long UNSET_DATA_CHECKSUM = -1;
private boolean isEmpty;
@@ -153,7 +154,7 @@ protected ContainerData(ContainerType type, long containerId,
this.originNodeId = originNodeId;
this.isEmpty = false;
this.checksum = ZERO_CHECKSUM;
- this.dataChecksum = 0;
+ this.dataChecksum = UNSET_DATA_CHECKSUM;
}
protected ContainerData(ContainerData source) {
@@ -538,13 +539,24 @@ public void computeAndSetContainerFileChecksum(Yaml yaml) throws IOException {
}
public void setDataChecksum(long dataChecksum) {
+ if (dataChecksum < 0) {
+ throw new IllegalArgumentException("Data checksum cannot be set to a negative number.");
+ }
this.dataChecksum = dataChecksum;
}
public long getDataChecksum() {
+ // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class.
+ if (needsDataChecksum()) {
+ return 0;
+ }
return dataChecksum;
}
+ public boolean needsDataChecksum() {
+ return dataChecksum == UNSET_DATA_CHECKSUM;
+ }
+
/**
* Returns a ProtoBuf Message from ContainerData.
*
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index 08fc949ea72..425729a6cc0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -23,6 +23,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.Message;
+import jakarta.annotation.Nullable;
import java.io.IOException;
import java.time.Clock;
import java.util.ArrayList;
@@ -43,12 +44,13 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.utils.ContainerLogger;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.metadata.ContainerCreateInfo;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,26 +69,28 @@ public class ContainerSet implements Iterable<Container<?>> {
new ConcurrentSkipListMap<>();
private final Clock clock;
private long recoveringTimeout;
- private final Table<ContainerID, String> containerIdsTable;
+ @Nullable
+ private final WitnessedContainerMetadataStore containerMetadataStore;
// Handler that will be invoked when a scan of a container in this set is requested.
- private OnDemandContainerDataScanner containerScanner;
+ private OnDemandContainerScanner containerScanner;
public static ContainerSet newReadOnlyContainerSet(long recoveringTimeout) {
return new ContainerSet(null, recoveringTimeout);
}
- public static ContainerSet newRwContainerSet(Table<ContainerID, String> containerIdsTable, long recoveringTimeout) {
- Objects.requireNonNull(containerIdsTable, "containerIdsTable == null");
- return new ContainerSet(containerIdsTable, recoveringTimeout);
+ public static ContainerSet newRwContainerSet(
+ WitnessedContainerMetadataStore metadataStore, long recoveringTimeout) {
+ Objects.requireNonNull(metadataStore, "WitnessedContainerMetadataStore == null");
+ return new ContainerSet(metadataStore, recoveringTimeout);
}
- private ContainerSet(Table<ContainerID, String> continerIdsTable, long recoveringTimeout) {
- this(continerIdsTable, recoveringTimeout, null);
+ private ContainerSet(WitnessedContainerMetadataStore containerMetadataStore, long recoveringTimeout) {
+ this(containerMetadataStore, recoveringTimeout, null);
}
- ContainerSet(Table<ContainerID, String> continerIdsTable, long recoveringTimeout, Clock clock) {
+ ContainerSet(WitnessedContainerMetadataStore containerMetadataStore, long recoveringTimeout, Clock clock) {
this.clock = clock != null ? clock : Clock.systemUTC();
- this.containerIdsTable = continerIdsTable;
+ this.containerMetadataStore = containerMetadataStore;
this.recoveringTimeout = recoveringTimeout;
}
@@ -131,7 +135,7 @@ public void ensureContainerNotMissing(long containerId, State state) throws Stor
/**
* @param scanner The scanner instance will be invoked when a scan of a container in this set is requested.
*/
- public void registerOnDemandScanner(OnDemandContainerDataScanner scanner) {
+ public void registerOnDemandScanner(OnDemandContainerScanner scanner) {
this.containerScanner = scanner;
}
@@ -140,11 +144,11 @@ public void registerOnDemandScanner(OnDemandContainerDataScanner scanner) {
* exist in the set.
* @param containerID The container in this set to scan.
*/
- public void scanContainer(long containerID) {
+ public void scanContainer(long containerID, String reasonForScan) {
if (containerScanner != null) {
Container<?> container = getContainer(containerID);
if (container != null) {
- containerScanner.scanContainer(container);
+ containerScanner.scanContainer(container, reasonForScan);
} else {
LOG.warn("Request to scan container {} which was not found in the container set", containerID);
}
@@ -156,11 +160,11 @@ public void scanContainer(long containerID) {
* This is a no-op if no scanner is registered or the container does not exist in the set.
* @param containerID The container in this set to scan.
*/
- public void scanContainerWithoutGap(long containerID) {
+ public void scanContainerWithoutGap(long containerID, String reasonForScan) {
if (containerScanner != null) {
Container<?> container = getContainer(containerID);
if (container != null) {
- containerScanner.scanContainerWithoutGap(container);
+ containerScanner.scanContainerWithoutGap(container, reasonForScan);
} else {
LOG.warn("Request to scan container {} which was not found in the container set", containerID);
}
@@ -188,13 +192,7 @@ private boolean addContainer(Container<?> container, boolean overwrite) throws
LOG.debug("Container with container Id {} is added to containerMap",
containerId);
}
- try {
- if (containerIdsTable != null) {
- containerIdsTable.put(ContainerID.valueOf(containerId), containerState.toString());
- }
- } catch (IOException e) {
- throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION);
- }
+ updateContainerIdTable(containerId, containerState);
missingContainerSet.remove(containerId);
if (container.getContainerData().getState() == RECOVERING) {
recoveringContainerMap.put(
@@ -209,6 +207,17 @@ private boolean addContainer(Container<?> container, boolean overwrite) throws
}
}
+ private void updateContainerIdTable(long containerId, State containerState) throws StorageContainerException {
+ if (null != containerMetadataStore) {
+ try {
+ containerMetadataStore.getContainerCreateInfoTable().put(ContainerID.valueOf(containerId),
+ ContainerCreateInfo.valueOf(containerState));
+ } catch (IOException e) {
+ throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION);
+ }
+ }
+ }
+
/**
* Update Container to container map.
* @param container container to be added
@@ -293,13 +302,7 @@ private boolean removeContainer(long containerId, boolean markMissing, boolean r
}
Container<?> removed = containerMap.remove(containerId);
if (removeFromDB) {
- try {
- if (containerIdsTable != null) {
- containerIdsTable.delete(ContainerID.valueOf(containerId));
- }
- } catch (IOException e) {
- throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION);
- }
+ deleteFromContainerTable(containerId);
}
if (removed == null) {
LOG.debug("Container with containerId {} is not present in " +
@@ -312,6 +315,16 @@ private boolean removeContainer(long containerId, boolean markMissing, boolean r
}
}
+ private void deleteFromContainerTable(long containerId) throws StorageContainerException {
+ if (null != containerMetadataStore) {
+ try {
+ containerMetadataStore.getContainerCreateInfoTable().delete(ContainerID.valueOf(containerId));
+ } catch (IOException e) {
+ throw new StorageContainerException(e, ContainerProtos.Result.IO_EXCEPTION);
+ }
+ }
+ }
+
/**
* Removes the Recovering Container matching with specified containerId.
* @param containerId ID of the container to remove.
@@ -526,10 +539,6 @@ public Set<Long> getMissingContainerSet() {
return missingContainerSet;
}
- public Table<ContainerID, String> getContainerIdsTable() {
- return containerIdsTable;
- }
-
/**
* Builds the missing container set by taking a diff between total no
* containers actually found and number of containers which actually
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 3c4b39de50e..ff6e199db29 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -434,7 +434,7 @@ && getMissingContainerSet().contains(containerID)) {
// Create a specific exception that signals for on demand scanning
// and move this general scan to where it is more appropriate.
// Add integration tests to test the full functionality.
- containerSet.scanContainer(containerID);
+ containerSet.scanContainer(containerID, result.name());
audit(action, eventType, msg, dispatcherContext, AuditEventStatus.FAILURE,
new Exception(responseProto.getMessage()));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index 7d1c0543d0b..6d2b9283147 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -258,13 +258,13 @@ public class DatanodeConfiguration extends ReconfigurableConfig {
private long blockDeleteMaxLockWaitTimeoutMs = Duration.ofMillis(100).toMillis();
@Config(key = "block.deleting.limit.per.interval",
- defaultValue = "5000",
+ defaultValue = "20000",
reconfigurable = true,
type = ConfigType.INT,
- tags = { ConfigTag.SCM, ConfigTag.DELETION },
+ tags = { ConfigTag.SCM, ConfigTag.DELETION, DATANODE },
description = "Number of blocks to be deleted in an interval."
)
- private int blockLimitPerInterval = 5000;
+ private int blockLimitPerInterval = 20000;
@Config(key = "block.deleting.max.lock.holding.time",
defaultValue = "1s",
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index dbc6b72968f..c4887bf917e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -333,6 +333,10 @@ public DeleteBlockTransactionExecutionResult call() {
"container={}, TXID={}", tx.getContainerID(), tx.getTxID(), e);
Thread.currentThread().interrupt();
txResultBuilder.setContainerID(containerId).setSuccess(false);
+ } catch (Exception e) {
+ LOG.error("Unexpected exception while deleting blocks for " +
+ "container={}, TXID={}", tx.getContainerID(), tx.getTxID(), e);
+ txResultBuilder.setContainerID(containerId).setSuccess(false);
}
return new DeleteBlockTransactionExecutionResult(
txResultBuilder.build(), lockAcquisitionFailed);
@@ -480,8 +484,10 @@ public void handleTasksResults(
try {
DeleteBlockTransactionExecutionResult result = f.get();
handler.accept(result);
- } catch (InterruptedException | ExecutionException e) {
+ } catch (ExecutionException e) {
LOG.error("task failed.", e);
+ } catch (InterruptedException e) {
+ LOG.error("task interrupted.", e);
Thread.currentThread().interrupt();
}
});
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
index f6663bfc438..d7c662aa9d7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
@@ -95,11 +95,8 @@ public void handle(SCMCommand<?> command, OzoneContainer container,
DatanodeDetails dni = context.getParent().getDatanodeDetails();
HddsProtos.NodeOperationalState state =
setNodeCmdProto.getNodeOperationalState();
- dni.setPersistedOpState(state);
- dni.setPersistedOpStateExpiryEpochSec(
- setNodeCmd.getStateExpiryEpochSeconds());
try {
- persistDatanodeDetails(dni);
+ persistUpdatedDatanodeDetails(dni, state, setNodeCmd.getStateExpiryEpochSeconds());
} catch (IOException ioe) {
LOG.error("Failed to persist the datanode state", ioe);
// TODO - this should probably be raised, but it will break the command
@@ -115,6 +112,17 @@ public void handle(SCMCommand<?> command, OzoneContainer container,
this.opsLatencyMs.add(Time.monotonicNow() - startTime);
}
+ private void persistUpdatedDatanodeDetails(
+ DatanodeDetails dnDetails, HddsProtos.NodeOperationalState state, long stateExpiryEpochSeconds)
+ throws IOException {
+ DatanodeDetails persistedDni = new DatanodeDetails(dnDetails);
+ persistedDni.setPersistedOpState(state);
+ persistedDni.setPersistedOpStateExpiryEpochSec(stateExpiryEpochSeconds);
+ persistDatanodeDetails(persistedDni);
+ dnDetails.setPersistedOpState(state);
+ dnDetails.setPersistedOpStateExpiryEpochSec(stateExpiryEpochSeconds);
+ }
+
// TODO - this duplicates code in HddsDatanodeService and InitDatanodeState
// Need to refactor.
private void persistDatanodeDetails(DatanodeDetails dnDetails)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index 0c72b0a6f91..d6f404dd17e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -289,6 +289,7 @@ public void cleanDeletedContainerDir() {
@Override
public synchronized VolumeCheckResult check(@Nullable Boolean unused)
throws Exception {
+ volumeInfoMetrics.incNumScans();
checkVolumeUsages();
VolumeCheckResult result = super.check(unused);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 5966e471b99..c6caf47d301 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -481,6 +481,10 @@ public StorageLocationReport[] getStorageReport() {
}
}
+ public StorageVolume.VolumeType getVolumeType() {
+ return volumeType;
+ }
+
public double getIdealUsage() {
long totalCapacity = 0L, totalFree = 0L;
for (StorageVolume volume: volumeMap.values()) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
index b779884a0c2..b80e6e23790 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java
@@ -65,10 +65,7 @@ public class StorageVolumeChecker {
private AsyncChecker<Boolean, VolumeCheckResult> delegateChecker;
- private final AtomicLong numVolumeChecks = new AtomicLong(0);
- private final AtomicLong numAllVolumeChecks = new AtomicLong(0);
- private final AtomicLong numAllVolumeSetsChecks = new AtomicLong(0);
- private final AtomicLong numSkippedChecks = new AtomicLong(0);
+ private final StorageVolumeScannerMetrics metrics;
/**
* Max allowed time for a disk check in milliseconds. If the check
@@ -108,6 +105,8 @@ public class StorageVolumeChecker {
public StorageVolumeChecker(ConfigurationSource conf, Timer timer,
String threadNamePrefix) {
+ metrics = StorageVolumeScannerMetrics.create();
+
this.timer = timer;
dnConf = conf.getObject(DatanodeConfiguration.class);
@@ -165,7 +164,7 @@ public synchronized void registerVolumeSet(VolumeSet volumeSet) {
public synchronized void checkAllVolumeSets() {
final long gap = timer.monotonicNow() - lastAllVolumeSetsCheckComplete;
if (gap < minDiskCheckGapMs) {
- numSkippedChecks.incrementAndGet();
+ metrics.incNumIterationsSkipped();
if (LOG.isTraceEnabled()) {
LOG.trace(
"Skipped checking all volumes, time since last check {} is less " +
@@ -176,12 +175,29 @@ public synchronized void checkAllVolumeSets() {
}
try {
+ long totalVolumesScanned = 0;
for (VolumeSet volSet : registeredVolumeSets) {
+ long volumeCount = volSet.getVolumesList().size();
+ if (volSet instanceof MutableVolumeSet) {
+ StorageVolume.VolumeType type = ((MutableVolumeSet) volSet).getVolumeType();
+ switch (type) {
+ case DATA_VOLUME:
+ metrics.incNumDataVolumeScans(volumeCount);
+ break;
+ case META_VOLUME:
+ metrics.incNumMetadataVolumeScans(volumeCount);
+ break;
+ default:
+ LOG.warn("Unknown volume type: {}", type);
+ break;
+ }
+ }
volSet.checkAllVolumes(this);
+ totalVolumesScanned += volSet.getVolumesList().size();
}
-
+ metrics.setNumVolumesScannedInLastIteration(totalVolumesScanned);
+ metrics.incNumScanIterations();
lastAllVolumeSetsCheckComplete = timer.monotonicNow();
- numAllVolumeSetsChecks.incrementAndGet();
} catch (IOException e) {
LOG.warn("Exception while checking disks", e);
}
@@ -219,6 +235,9 @@ public Set<? extends StorageVolume> checkAllVolumes(
numVolumes, (ignored1, ignored2) -> latch.countDown()),
MoreExecutors.directExecutor());
} else {
+ if (v instanceof HddsVolume) {
+ ((HddsVolume) v).getVolumeInfoStats().incNumScansSkipped();
+ }
if (numVolumes.decrementAndGet() == 0) {
latch.countDown();
}
@@ -232,7 +251,6 @@ public Set<? extends StorageVolume> checkAllVolumes(
maxAllowedTimeForCheckMs);
}
- numAllVolumeChecks.incrementAndGet();
synchronized (this) {
// All volumes that have not been detected as healthy should be
// considered failed. This is a superset of 'failedVolumes'.
@@ -277,7 +295,6 @@ public boolean checkVolume(final StorageVolume volume, Callback callback) {
Optional<ListenableFuture<VolumeCheckResult>> olf =
delegateChecker.schedule(volume, null);
if (olf.isPresent()) {
- numVolumeChecks.incrementAndGet();
Futures.addCallback(olf.get(),
new ResultHandler(volume,
ConcurrentHashMap.newKeySet(), ConcurrentHashMap.newKeySet(),
@@ -401,6 +418,7 @@ public void shutdownAndWait(int gracePeriod, TimeUnit timeUnit) {
periodicDiskChecker.cancel(true);
diskCheckerservice.shutdownNow();
checkVolumeResultHandlerExecutorService.shutdownNow();
+ metrics.unregister();
try {
delegateChecker.shutdownAndWait(gracePeriod, timeUnit);
} catch (InterruptedException e) {
@@ -422,32 +440,8 @@ void setDelegateChecker(
delegateChecker = testDelegate;
}
- /**
- * Return the number of {@link #checkVolume} invocations.
- */
- public long getNumVolumeChecks() {
- return numVolumeChecks.get();
- }
-
- /**
- * Return the number of {@link #checkAllVolumes(Collection)} ()} invocations.
- */
- public long getNumAllVolumeChecks() {
- return numAllVolumeChecks.get();
- }
-
- /**
- * Return the number of {@link #checkAllVolumeSets()} invocations.
- */
- public long getNumAllVolumeSetsChecks() {
- return numAllVolumeSetsChecks.get();
- }
-
- /**
- * Return the number of checks skipped because the minimum gap since the
- * last check had not elapsed.
- */
- public long getNumSkippedChecks() {
- return numSkippedChecks.get();
+ @VisibleForTesting
+ public StorageVolumeScannerMetrics getMetrics() {
+ return metrics;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeScannerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeScannerMetrics.java
new file mode 100644
index 00000000000..88e45927b3e
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeScannerMetrics.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.volume;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+
+/**
+ * This class captures the Storage Volume Scanner Metrics.
+ **/
+@InterfaceAudience.Private
+@Metrics(about = "Storage Volume Scanner Metrics", context = "dfs")
+public class StorageVolumeScannerMetrics {
+ public static final String SOURCE_NAME = StorageVolumeScannerMetrics.class.getSimpleName();
+
+ @Metric("number of volumes scanned in the last iteration")
+ private MutableGaugeLong numVolumesScannedInLastIteration;
+
+ @Metric("number of iterations the volume scanner has completed since the last restart")
+ private MutableCounterLong numScanIterations;
+
+ @Metric("number of data volume scans since the last restart")
+ private MutableCounterLong numDataVolumeScans;
+
+ @Metric("number of metadata volume scans since the last restart")
+ private MutableCounterLong numMetadataVolumeScans;
+
+ @Metric("number of volume scanner iterations skipped because the minimum gap " +
+ "since the last iteration had not elapsed")
+ private MutableCounterLong numIterationsSkipped;
+
+ public StorageVolumeScannerMetrics() {
+ }
+
+ public static StorageVolumeScannerMetrics create() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register(SOURCE_NAME, "Storage Volume Scanner Metrics", new StorageVolumeScannerMetrics());
+ }
+
+ /**
+ * Return the number of volumes scanned in the last iteration.
+ */
+ public long getNumVolumesScannedInLastIteration() {
+ return numVolumesScannedInLastIteration.value();
+ }
+
+ public void setNumVolumesScannedInLastIteration(long value) {
+ numVolumesScannedInLastIteration.set(value);
+ }
+
+ /**
+ * Return the number of iterations the volume scanner has completed since the last restart.
+ */
+ public long getNumScanIterations() {
+ return numScanIterations.value();
+ }
+
+ public void incNumScanIterations() {
+ numScanIterations.incr();
+ }
+
+ /**
+ * Return the number of data volumes scanned since the last restart.
+ */
+ public long getNumDataVolumeScans() {
+ return numDataVolumeScans.value();
+ }
+
+ public void incNumDataVolumeScans(long count) {
+ numDataVolumeScans.incr(count);
+ }
+
+ /**
+ * Return the number of metadata volumes scanned since the last restart.
+ */
+ public long getNumMetadataVolumeScans() {
+ return numMetadataVolumeScans.value();
+ }
+
+ public void incNumMetadataVolumeScans(long count) {
+ numMetadataVolumeScans.incr(count);
+ }
+
+ /**
+ * Return the number of volume scanner iterations skipped because the minimum gap
+ * since the last iteration had not elapsed.
+ */
+ public long getNumIterationsSkipped() {
+ return numIterationsSkipped.value();
+ }
+
+ public void incNumIterationsSkipped() {
+ numIterationsSkipped.incr();
+ }
+
+ public void unregister() {
+ DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
index c8cb2a52215..8a6addf306a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -63,6 +64,12 @@ public class VolumeInfoMetrics implements MetricsSource {
@Metric("Volume available space is insufficient")
private MutableGaugeInt availableSpaceInsufficient;
+ @Metric("Number of times the volume is scanned")
+ private MutableCounterLong numScans;
+
+ @Metric("Number of scans skipped for the volume")
+ private MutableCounterLong numScansSkipped;
+
/**
* @param identifier Typically, path to volume root. E.g. /data/hdds
*/
@@ -156,6 +163,22 @@ public long getContainers() {
return volume.getContainers();
}
+ public long getNumScans() {
+ return numScans.value();
+ }
+
+ public void incNumScans() {
+ numScans.incr();
+ }
+
+ public long getNumScansSkipped() {
+ return numScansSkipped.value();
+ }
+
+ public void incNumScansSkipped() {
+ numScansSkipped.incr();
+ }
+
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
MetricsRecordBuilder builder = collector.addRecord(metricsSourceName);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 022764c227d..0c398d24498 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_COUNT;
import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_BYTES_USED;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
import static org.apache.hadoop.ozone.OzoneConsts.DELETE_TRANSACTION_KEY;
@@ -424,6 +425,10 @@ public String getPendingDeleteBlockCountKey() {
return formatKey(PENDING_DELETE_BLOCK_COUNT);
}
+ public String getContainerDataChecksumKey() {
+ return formatKey(CONTAINER_DATA_CHECKSUM);
+ }
+
public String getDeletingBlockKeyPrefix() {
return formatKey(DELETING_KEY_PREFIX);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 21763382275..9a2f5e48503 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -63,6 +63,7 @@
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Striped;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -81,7 +82,6 @@
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
-import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.locks.Lock;
@@ -761,14 +761,33 @@ ContainerCommandResponseProto handleGetContainerChecksumInfo(
try {
checksumTree = checksumManager.getContainerChecksumInfo(containerData);
} catch (IOException ex) {
- // For file not found or other inability to read the file, return an error to the client.
- LOG.error("Error occurred when reading checksum file for container {}", containerData.getContainerID(), ex);
- return ContainerCommandResponseProto.newBuilder()
- .setCmdType(request.getCmdType())
- .setTraceID(request.getTraceID())
- .setResult(IO_EXCEPTION)
- .setMessage(ex.getMessage())
- .build();
+ // Only build from metadata if the file doesn't exist
+ if (ex instanceof FileNotFoundException) {
+ try {
+ LOG.info("Checksum tree file not found for container {}. Building merkle tree from container metadata.",
+ containerData.getContainerID());
+ ContainerProtos.ContainerChecksumInfo checksumInfo = updateAndGetContainerChecksumFromMetadata(kvContainer);
+ checksumTree = checksumInfo.toByteString();
+ } catch (IOException metadataEx) {
+ LOG.error("Failed to build merkle tree from metadata for container {}",
+ containerData.getContainerID(), metadataEx);
+ return ContainerCommandResponseProto.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setTraceID(request.getTraceID())
+ .setResult(IO_EXCEPTION)
+ .setMessage("Failed to get or build merkle tree: " + metadataEx.getMessage())
+ .build();
+ }
+ } else {
+ // For other inability to read the file, return an error to the client.
+ LOG.error("Error occurred when reading checksum file for container {}", containerData.getContainerID(), ex);
+ return ContainerCommandResponseProto.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setTraceID(request.getTraceID())
+ .setResult(IO_EXCEPTION)
+ .setMessage(ex.getMessage())
+ .build();
+ }
}
return getGetContainerMerkleTreeResponse(request, checksumTree);
@@ -1368,7 +1387,6 @@ public void markContainerForClose(Container container)
} finally {
container.writeUnlock();
}
- updateContainerChecksumFromMetadataIfNeeded(container);
ContainerLogger.logClosing(container.getContainerData());
sendICR(container);
}
@@ -1383,7 +1401,7 @@ public void updateContainerChecksum(Container container, ContainerMerkleTreeWrit
* Write the merkle tree for this container using the existing checksum metadata only. The data is not read or
* validated by this method, so it is expected to run quickly.
* <p>
- * If a checksum file already exists on the disk, this method will do nothing. The existing file would have either
+ * If a data checksum for the container already exists, this method does nothing. The existing value would have either
* been made from the metadata or data itself so there is no need to recreate it from the metadata. This method
* does not send an ICR with the updated checksum info.
* <p>
@@ -1391,7 +1409,7 @@ public void updateContainerChecksum(Container container, ContainerMerkleTreeWrit
* @param container The container which will have a tree generated.
*/
private void updateContainerChecksumFromMetadataIfNeeded(Container container) {
- if (ContainerChecksumTreeManager.checksumFileExist(container)) {
+ if (!container.getContainerData().needsDataChecksum()) {
return;
}
@@ -1409,7 +1427,8 @@ private void updateContainerChecksumFromMetadataIfNeeded(Container container) {
* This method does not send an ICR with the updated checksum info.
* @param container - Container for which the container merkle tree needs to be updated.
*/
- private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromMetadata(
+ @VisibleForTesting
+ public ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromMetadata(
KeyValueContainer container) throws IOException {
ContainerMerkleTreeWriter merkleTree = new ContainerMerkleTreeWriter();
try (DBHandle dbHandle = BlockUtils.getDB(container.getContainerData(), conf);
@@ -1429,30 +1448,41 @@ private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromM
private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksum(Container container,
ContainerMerkleTreeWriter treeWriter, boolean sendICR) throws IOException {
- ContainerData containerData = container.getContainerData();
+ KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
// Attempt to write the new data checksum to disk. If persisting this fails, keep using the original data
// checksum to prevent divergence from what SCM sees in the ICR vs what datanode peers will see when pulling the
// merkle tree.
long originalDataChecksum = containerData.getDataChecksum();
+ boolean hadDataChecksum = !containerData.needsDataChecksum();
ContainerProtos.ContainerChecksumInfo updateChecksumInfo = checksumManager.writeContainerDataTree(containerData,
treeWriter);
long updatedDataChecksum = updateChecksumInfo.getContainerMerkleTree().getDataChecksum();
if (updatedDataChecksum != originalDataChecksum) {
containerData.setDataChecksum(updatedDataChecksum);
- String message =
- "Container data checksum updated from " + checksumToString(originalDataChecksum) + " to " +
- checksumToString(updatedDataChecksum);
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+ // This value is only used during the datanode startup. If the update fails, then it's okay as the merkle tree
+ // and in-memory checksum will still be the same. This will be updated the next time we update the tree.
+ // Either scanner or reconciliation will update the checksum.
+ dbHandle.getStore().getMetadataTable().put(containerData.getContainerDataChecksumKey(), updatedDataChecksum);
+ } catch (IOException e) {
+ LOG.error("Failed to update container data checksum in RocksDB for container {}. " +
+ "Leaving the original checksum in RocksDB: {}", containerData.getContainerID(),
+ checksumToString(originalDataChecksum), e);
+ }
+
if (sendICR) {
sendICR(container);
}
- if (ContainerChecksumTreeManager.hasContainerChecksumFile(containerData)) {
+
+ String message = "Container " + containerData.getContainerID() + " data checksum updated from " +
+ checksumToString(originalDataChecksum) + " to " + checksumToString(updatedDataChecksum);
+ if (hadDataChecksum) {
LOG.warn(message);
ContainerLogger.logChecksumUpdated(containerData, originalDataChecksum);
} else {
- // If this is the first time the scanner has run with the feature to generate a checksum file, don't
- // log a warning for the checksum update.
+ // If this is the first time the checksum is being generated, don't log a warning about updating the checksum.
LOG.debug(message);
}
}
@@ -1465,7 +1495,6 @@ public void markContainerUnhealthy(Container container, ScanResult reason)
container.writeLock();
long containerID = 0L;
try {
- containerID = container.getContainerData().getContainerID();
if (container.getContainerState() == State.UNHEALTHY) {
LOG.debug("Call to mark already unhealthy container {} as unhealthy",
containerID);
@@ -1590,12 +1619,9 @@ public void reconcileContainer(DNContainerOperationClient dnClient, Container<?>
long containerID = containerData.getContainerID();
// Obtain the original checksum info before reconciling with any peers.
- Optional<ContainerProtos.ContainerChecksumInfo> optionalChecksumInfo = checksumManager.read(containerData);
- ContainerProtos.ContainerChecksumInfo originalChecksumInfo;
- if (optionalChecksumInfo.isPresent()) {
- originalChecksumInfo = optionalChecksumInfo.get();
- } else {
- // Try creating the checksum info from RocksDB metadata if it is not present.
+ ContainerProtos.ContainerChecksumInfo originalChecksumInfo = checksumManager.read(containerData);
+ if (!originalChecksumInfo.hasContainerMerkleTree()) {
+ // Try creating the merkle tree from RocksDB metadata if it is not present.
originalChecksumInfo = updateAndGetContainerChecksumFromMetadata(kvContainer);
}
// This holds our current most up-to-date checksum info that we are using for the container.
@@ -1733,7 +1759,8 @@ containerID, peer, checksumToString(previousDataChecksum), checksumToString(late
}
// Trigger on demand scanner, which will build the merkle tree based on the newly ingested data.
- containerSet.scanContainerWithoutGap(containerID);
+ containerSet.scanContainerWithoutGap(containerID,
+ "Container reconciliation");
sendICR(container);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java
index d42eb208e11..9ab8a8c5806 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java
@@ -24,6 +24,7 @@
import static org.apache.hadoop.hdds.utils.Archiver.readEntry;
import static org.apache.hadoop.hdds.utils.Archiver.tar;
import static org.apache.hadoop.hdds.utils.Archiver.untar;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM_EXTENSION;
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
import java.io.File;
@@ -44,7 +45,9 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
@@ -92,8 +95,9 @@ public byte[] unpackContainerData(Container<KeyValueContainerData> container,
}
Path dbRoot = getDbPath(containerUntarDir, containerData);
- Path chunksRoot = getChunkPath(containerUntarDir, containerData);
- byte[] descriptorFileContent = innerUnpack(input, dbRoot, chunksRoot);
+ Path chunksRoot = getChunkPath(containerUntarDir);
+ Path tempContainerMetadataPath = getTempContainerMetadataPath(containerUntarDir, containerData);
+ byte[] descriptorFileContent = innerUnpack(input, dbRoot, chunksRoot, tempContainerMetadataPath);
if (!Files.exists(destContainerDir)) {
Files.createDirectories(destContainerDir);
@@ -111,9 +115,6 @@ public byte[] unpackContainerData(Container<KeyValueContainerData> container,
// Before the atomic move, the destination dir is empty and doesn't have a metadata directory.
// Writing the .container file will fail as the metadata dir doesn't exist.
// So we instead save the container file to the containerUntarDir.
- Path containerMetadataPath = Paths.get(container.getContainerData().getMetadataPath());
- Path tempContainerMetadataPath = Paths.get(containerUntarDir.toString(),
- containerMetadataPath.getName(containerMetadataPath.getNameCount() - 1).toString());
persistCustomContainerState(container, descriptorFileContent, State.RECOVERING, tempContainerMetadataPath);
Files.move(containerUntarDir, destContainerDir,
StandardCopyOption.ATOMIC_MOVE,
@@ -146,6 +147,11 @@ public void pack(Container<KeyValueContainerData> container,
includeFile(container.getContainerFile(), CONTAINER_FILE_NAME,
archiveOutput);
+ File containerChecksumFile = ContainerChecksumTreeManager.getContainerChecksumFile(containerData);
+ if (containerChecksumFile.exists()) {
+ includeFile(containerChecksumFile, containerChecksumFile.getName(), archiveOutput);
+ }
+
includePath(getDbPath(containerData), DB_DIR_NAME,
archiveOutput);
@@ -202,11 +208,20 @@ public static Path getDbPath(Path baseDir,
}
}
- public static Path getChunkPath(Path baseDir,
- KeyValueContainerData containerData) {
+ public static Path getChunkPath(Path baseDir) {
return KeyValueContainerLocationUtil.getChunksLocationPath(baseDir.toString()).toPath();
}
+ private Path getContainerMetadataPath(ContainerData containerData) {
+ return Paths.get(containerData.getMetadataPath());
+ }
+
+ private Path getTempContainerMetadataPath(Path containerUntarDir, ContainerData containerData) {
+ Path containerMetadataPath = getContainerMetadataPath(containerData);
+ return Paths.get(containerUntarDir.toString(),
+ containerMetadataPath.getName(containerMetadataPath.getNameCount() - 1).toString());
+ }
+
InputStream decompress(InputStream input) throws IOException {
return compression.wrap(input);
}
@@ -215,7 +230,7 @@ OutputStream compress(OutputStream output) throws IOException {
return compression.wrap(output);
}
- private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot)
+ private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot, Path metadataRoot)
throws IOException {
byte[] descriptorFileContent = null;
try (ArchiveInputStream<TarArchiveEntry> archiveInput = untar(decompress(input))) {
@@ -233,6 +248,10 @@ private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot)
.resolve(name.substring(CHUNKS_DIR_NAME.length() + 1));
extractEntry(entry, archiveInput, size, chunksRoot,
destinationPath);
+ } else if (name.endsWith(CONTAINER_DATA_CHECKSUM_EXTENSION)) {
+ Path destinationPath = metadataRoot.resolve(name);
+ extractEntry(entry, archiveInput, size, metadataRoot,
+ destinationPath);
} else if (CONTAINER_FILE_NAME.equals(name)) {
//Don't do anything. Container file should be unpacked in a
//separated step by unpackContainerDescriptor call.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index 0550ecad4a3..28b1711e980 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -26,7 +26,6 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -236,8 +235,8 @@ public static void parseKVContainerData(KeyValueContainerData kvContainerData,
File dbFile = KeyValueContainerLocationUtil.getContainerDBFile(
kvContainerData);
if (!dbFile.exists()) {
- LOG.error("Container DB file is missing for ContainerID {}. " +
- "Skipping loading of this container.", containerID);
+ LOG.error("Container DB file is missing at {} for ContainerID {}. " +
+ "Skipping loading of this container.", dbFile, containerID);
// Don't further process this container, as it is missing db file.
throw new IOException("Container DB file is missing for containerID "
+ containerID);
@@ -260,10 +259,10 @@ public static void parseKVContainerData(KeyValueContainerData kvContainerData,
DatanodeStore store = null;
try {
try {
- boolean readOnly = ContainerInspectorUtil.isReadOnly(
- ContainerProtos.ContainerType.KeyValueContainer);
- store = BlockUtils.getUncachedDatanodeStore(
- kvContainerData, config, readOnly);
+ // Open RocksDB in write mode, as it is required for container checksum updates and inspector repair operations.
+ // The method KeyValueContainerMetadataInspector.buildErrorAndRepair will determine if write access to the DB
+ // is permitted based on the mode.
+ store = BlockUtils.getUncachedDatanodeStore(kvContainerData, config, false);
} catch (IOException e) {
// If an exception is thrown, then it may indicate the RocksDB is
// already open in the container cache. As this code is only executed at
@@ -288,17 +287,25 @@ public static void parseKVContainerData(KeyValueContainerData kvContainerData,
}
}
- private static void populateContainerDataChecksum(KeyValueContainerData kvContainerData) {
+ private static void loadAndSetContainerDataChecksum(KeyValueContainerData kvContainerData,
+ Table<String, Long> metadataTable) {
if (kvContainerData.isOpen()) {
return;
}
try {
- Optional<ContainerChecksumInfo> optionalContainerChecksumInfo = ContainerChecksumTreeManager
- .readChecksumInfo(kvContainerData);
- if (optionalContainerChecksumInfo.isPresent()) {
- ContainerChecksumInfo containerChecksumInfo = optionalContainerChecksumInfo.get();
- kvContainerData.setDataChecksum(containerChecksumInfo.getContainerMerkleTree().getDataChecksum());
+ Long containerDataChecksum = metadataTable.get(kvContainerData.getContainerDataChecksumKey());
+ if (containerDataChecksum != null && kvContainerData.needsDataChecksum()) {
+ kvContainerData.setDataChecksum(containerDataChecksum);
+ return;
+ }
+
+ ContainerChecksumInfo containerChecksumInfo = ContainerChecksumTreeManager.readChecksumInfo(kvContainerData);
+ if (containerChecksumInfo != null && containerChecksumInfo.hasContainerMerkleTree()
+ && kvContainerData.needsDataChecksum()) {
+ containerDataChecksum = containerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+ kvContainerData.setDataChecksum(containerDataChecksum);
+ metadataTable.put(kvContainerData.getContainerDataChecksumKey(), containerDataChecksum);
}
} catch (IOException ex) {
LOG.warn("Failed to read checksum info for container {}", kvContainerData.getContainerID(), ex);
@@ -375,6 +382,8 @@ private static void populateContainerMetadata(
kvContainerData.markAsEmpty();
}
+ loadAndSetContainerDataChecksum(kvContainerData, metadataTable);
+
// Run advanced container inspection/repair operations if specified on
// startup. If this method is called but not as a part of startup,
// The inspectors will be unloaded and this will be a no-op.
@@ -382,7 +391,6 @@ private static void populateContainerMetadata(
// Load finalizeBlockLocalIds for container in memory.
populateContainerFinalizeBlock(kvContainerData, store);
- populateContainerDataChecksum(kvContainerData);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/ContainerCreateInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/ContainerCreateInfo.java
new file mode 100644
index 00000000000..ab7700c6ff3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/ContainerCreateInfo.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.metadata;
+
+import java.util.function.Supplier;
+import net.jcip.annotations.Immutable;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto3Codec;
+import org.apache.ratis.util.MemoizedSupplier;
+
+/**
+ * ContainerCreateInfo is a class that holds information about the state and other information on creation
+ * This class is immutable.
+ */
+@Immutable
+public final class ContainerCreateInfo {
+ private static final Codec<ContainerCreateInfo> CODEC = new DelegatedCodec<>(
+ Proto3Codec.get(ContainerProtos.ContainerCreateInfo.getDefaultInstance()),
+ ContainerCreateInfo::getFromProtobuf, ContainerCreateInfo::getProtobuf,
+ ContainerCreateInfo.class);
+
+ private final ContainerProtos.ContainerDataProto.State state;
+ private final Supplier<ContainerProtos.ContainerCreateInfo> proto;
+
+ public static Codec<ContainerCreateInfo> getCodec() {
+ return CODEC;
+ }
+
+ public static Codec<ContainerCreateInfo> getNewCodec() {
+ return CODEC;
+ }
+
+ private ContainerCreateInfo(ContainerProtos.ContainerDataProto.State state) {
+ this.state = state;
+ this.proto = MemoizedSupplier.valueOf(
+ () -> ContainerProtos.ContainerCreateInfo.newBuilder().setState(state).build());
+ }
+
+ /**
+ * Factory method for creation of ContainerCreateInfo.
+ * @param state State
+ * @return ContainerCreateInfo.
+ */
+ public static ContainerCreateInfo valueOf(final ContainerProtos.ContainerDataProto.State state) {
+ return new ContainerCreateInfo(state);
+ }
+
+ public ContainerProtos.ContainerCreateInfo getProtobuf() {
+ return proto.get();
+ }
+
+ public static ContainerCreateInfo getFromProtobuf(ContainerProtos.ContainerCreateInfo proto) {
+ return ContainerCreateInfo.valueOf(proto.getState());
+ }
+
+ public ContainerProtos.ContainerDataProto.State getState() {
+ return state;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerDBDefinition.java
index a1e76b19f43..f5869737acd 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerDBDefinition.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerDBDefinition.java
@@ -22,7 +22,6 @@
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.DBDefinition;
-import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.ozone.OzoneConsts;
/**
@@ -30,17 +29,17 @@
*/
public final class WitnessedContainerDBDefinition extends DBDefinition.WithMap {
- private static final String CONTAINER_IDS_TABLE_NAME = "containerIds";
+ private static final String CONTAINER_CREATE_INFO_TABLE_NAME = "ContainerCreateInfoTable";
- public static final DBColumnFamilyDefinition<ContainerID, String>
- CONTAINER_IDS_TABLE = new DBColumnFamilyDefinition<>(
- CONTAINER_IDS_TABLE_NAME,
+ public static final DBColumnFamilyDefinition<ContainerID, ContainerCreateInfo>
+ CONTAINER_CREATE_INFO_TABLE_DEF = new DBColumnFamilyDefinition<>(
+ CONTAINER_CREATE_INFO_TABLE_NAME,
ContainerID.getCodec(),
- StringCodec.get());
+ ContainerCreateInfo.getCodec());
private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
- CONTAINER_IDS_TABLE);
+ CONTAINER_CREATE_INFO_TABLE_DEF);
private static final WitnessedContainerDBDefinition INSTANCE = new WitnessedContainerDBDefinition();
@@ -62,7 +61,7 @@ public String getLocationConfigKey() {
return ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR;
}
- DBColumnFamilyDefinition<ContainerID, String> getContainerIdsTable() {
- return CONTAINER_IDS_TABLE;
+ DBColumnFamilyDefinition<ContainerID, ContainerCreateInfo> getContainerCreateInfoTableDef() {
+ return CONTAINER_CREATE_INFO_TABLE_DEF;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStore.java
index 815879a9ada..64d1b1abbd4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStore.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStore.java
@@ -29,5 +29,5 @@ public interface WitnessedContainerMetadataStore extends DBStoreManager {
*
* @return Table
*/
- Table<ContainerID, String> getContainerIdsTable();
+ Table<ContainerID, ContainerCreateInfo> getContainerCreateInfoTable();
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStoreImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStoreImpl.java
index 2e1e0386aab..a389d0497ff 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStoreImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/WitnessedContainerMetadataStoreImpl.java
@@ -22,13 +22,18 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.db.CodecException;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
/**
* Class for interacting with database in the master volume of a datanode.
@@ -36,7 +41,9 @@
public final class WitnessedContainerMetadataStoreImpl extends AbstractRDBStore<WitnessedContainerDBDefinition>
implements WitnessedContainerMetadataStore {
- private Table<ContainerID, String> containerIdsTable;
+ private Table<ContainerID, ContainerCreateInfo> containerCreateInfoTable;
+ private PreviousVersionTables previousVersionTables;
+
private static final ConcurrentMap<String, WitnessedContainerMetadataStore> INSTANCES =
new ConcurrentHashMap<>();
@@ -67,13 +74,53 @@ private WitnessedContainerMetadataStoreImpl(ConfigurationSource config, boolean
@Override
protected DBStore initDBStore(DBStoreBuilder dbStoreBuilder, ManagedDBOptions options, ConfigurationSource config)
throws RocksDatabaseException, CodecException {
+ previousVersionTables = new PreviousVersionTables();
+ previousVersionTables.addTables(dbStoreBuilder);
final DBStore dbStore = dbStoreBuilder.build();
- this.containerIdsTable = this.getDbDef().getContainerIdsTable().getTable(dbStore);
+ previousVersionTables.init(dbStore);
+ this.containerCreateInfoTable = this.getDbDef().getContainerCreateInfoTableDef().getTable(dbStore);
return dbStore;
}
@Override
- public Table<ContainerID, String> getContainerIdsTable() {
- return containerIdsTable;
+ public Table<ContainerID, ContainerCreateInfo> getContainerCreateInfoTable() {
+ if (!VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE)) {
+ return previousVersionTables.getContainerIdsTable();
+ }
+ return containerCreateInfoTable;
+ }
+
+ public PreviousVersionTables getPreviousVersionTables() {
+ return previousVersionTables;
+ }
+
+ /**
+ * this will hold old version tables required during upgrade, and these are initialized based on version only.
+ */
+ public static class PreviousVersionTables {
+ private static final String CONTAINER_IDS_STR_VAL_TABLE = "containerIds";
+ private Table<ContainerID, ContainerCreateInfo> containerIdsTable;
+
+ public PreviousVersionTables() {
+ }
+
+ public void addTables(DBStoreBuilder dbStoreBuilder) {
+ if (!VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE)) {
+ dbStoreBuilder.addTable(CONTAINER_IDS_STR_VAL_TABLE);
+ }
+ }
+
+ public void init(DBStore dbStore) throws RocksDatabaseException, CodecException {
+ if (!VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE)) {
+ this.containerIdsTable = dbStore.getTable(CONTAINER_IDS_STR_VAL_TABLE, ContainerID.getCodec(),
+ new DelegatedCodec<>(StringCodec.get(),
+ (strVal) -> ContainerCreateInfo.valueOf(ContainerProtos.ContainerDataProto.State.valueOf(strVal)),
+ (obj) -> obj.getState().name(), ContainerCreateInfo.class));
+ }
+ }
+
+ public Table<ContainerID, ContainerCreateInfo> getContainerIdsTable() {
+ return containerIdsTable;
+ }
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
index cb57fc1929f..ae634a8a62f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
@@ -54,24 +54,7 @@ public Iterator<Container<?>> getContainerIterator() {
@Override
public void scanContainer(Container<?> container)
throws IOException, InterruptedException {
- if (!scanHelper.shouldScanMetadata(container)) {
- return;
- }
-
- long containerID = container.getContainerData().getContainerID();
-
- MetadataScanResult result = container.scanMetaData();
- if (result.isDeleted()) {
- LOG.debug("Container [{}] has been deleted during the metadata scan.", containerID);
- return;
- }
- if (result.hasErrors()) {
- scanHelper.handleUnhealthyScanResult(containerID, result);
- }
-
- // Do not update the scan timestamp after the scan since this was just a
- // metadata scan, not a full data scan.
- metrics.incNumContainersScanned();
+ scanHelper.scanMetadata(container);
}
@Override
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
index 6cd7b80f02f..4c4a45c55d4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.slf4j.Logger;
@@ -63,7 +64,7 @@ public void scanData(Container<?> container, DataTransferThrottler throttler, Ca
}
ContainerData containerData = container.getContainerData();
long containerId = containerData.getContainerID();
- logScanStart(containerData);
+ logScanStart(containerData, "data");
DataScanResult result = container.scanData(throttler, canceler);
if (result.isDeleted()) {
@@ -75,7 +76,7 @@ public void scanData(Container<?> container, DataTransferThrottler throttler, Ca
log.warn("Failed to update container checksum after scan of container {}", containerId, ex);
}
if (result.hasErrors()) {
- handleUnhealthyScanResult(containerId, result);
+ handleUnhealthyScanResult(containerData, result);
}
metrics.incNumContainersScanned();
}
@@ -88,8 +89,34 @@ public void scanData(Container<?> container, DataTransferThrottler throttler, Ca
logScanCompleted(containerData, now);
}
- public void handleUnhealthyScanResult(long containerID, ScanResult result) throws IOException {
+ public void scanMetadata(Container<?> container)
+ throws IOException, InterruptedException {
+ if (!shouldScanMetadata(container)) {
+ return;
+ }
+ ContainerData containerData = container.getContainerData();
+ long containerId = containerData.getContainerID();
+ logScanStart(containerData, "only metadata");
+
+ MetadataScanResult result = container.scanMetaData();
+ if (result.isDeleted()) {
+ log.debug("Container [{}] has been deleted during metadata scan.", containerId);
+ return;
+ }
+ if (result.hasErrors()) {
+ handleUnhealthyScanResult(containerData, result);
+ }
+ Instant now = Instant.now();
+ // Do not update the scan timestamp after the scan since this was just a
+ // metadata scan, not a full data scan.
+ metrics.incNumContainersScanned();
+ // Even if the container was deleted, mark the scan as completed since we already logged it as starting.
+ logScanCompleted(containerData, now);
+ }
+
+ public void handleUnhealthyScanResult(ContainerData containerData, ScanResult result) throws IOException {
+ long containerID = containerData.getContainerID();
log.error("Corruption detected in container [{}]. Marking it UNHEALTHY. {}", containerID, result);
if (log.isDebugEnabled()) {
StringBuilder allErrorString = new StringBuilder();
@@ -104,6 +131,23 @@ public void handleUnhealthyScanResult(long containerID, ScanResult result) throw
boolean containerMarkedUnhealthy = controller.markContainerUnhealthy(containerID, result);
if (containerMarkedUnhealthy) {
metrics.incNumUnHealthyContainers();
+ // triggering a volume scan for the unhealthy container
+ triggerVolumeScan(containerData);
+ }
+ }
+
+ public void triggerVolumeScan(ContainerData containerData) {
+ HddsVolume volume = containerData.getVolume();
+ if (volume != null && !volume.isFailed()) {
+ log.info("Triggering scan of volume [{}] with unhealthy container [{}]",
+ volume, containerData.getContainerID());
+ StorageVolumeUtil.onFailure(volume);
+ } else if (volume == null) {
+ log.warn("Cannot trigger volume scan for container {} since its volume is null",
+ containerData.getContainerID());
+ } else {
+ log.debug("Skipping volume scan for container {} since its volume {} has failed.",
+ containerData.getContainerID(), volume);
}
}
@@ -147,12 +191,12 @@ private boolean recentlyScanned(ContainerData containerData) {
return recentlyScanned;
}
- private void logScanStart(ContainerData containerData) {
+ private void logScanStart(ContainerData containerData, String scanType) {
if (log.isDebugEnabled()) {
Optional<Instant> scanTimestamp = containerData.lastDataScanTime();
Object lastScanTime = scanTimestamp.map(ts -> "at " + ts).orElse("never");
- log.debug("Scanning container {}, last scanned {}",
- containerData.getContainerID(), lastScanTime);
+ log.debug("Scanning {} of container {}, last scanned {}",
+ scanType, containerData.getContainerID(), lastScanTime);
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
similarity index 78%
rename from hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java
rename to hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
index a85358406bd..99dad8f2e1f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
@@ -32,10 +32,12 @@
/**
* Class for performing on demand scans of containers.
+ * Note: [OPEN] containers are scanned only for metadata,
+ * [CLOSED, QUASI_CLOSED] containers are scanned for metadata and data.
*/
-public final class OnDemandContainerDataScanner {
+public final class OnDemandContainerScanner {
private static final Logger LOG =
- LoggerFactory.getLogger(OnDemandContainerDataScanner.class);
+ LoggerFactory.getLogger(OnDemandContainerScanner.class);
private final ExecutorService scanExecutor;
private final DataTransferThrottler throttler;
@@ -46,7 +48,7 @@ public final class OnDemandContainerDataScanner {
private final ContainerScanHelper scannerHelper;
private final ContainerScanHelper scannerHelperWithoutGap;
- public OnDemandContainerDataScanner(
+ public OnDemandContainerScanner(
ContainerScannerConfiguration conf, ContainerController controller) {
throttler = new DataTransferThrottler(
conf.getOnDemandBandwidthPerVolume());
@@ -63,8 +65,8 @@ public OnDemandContainerDataScanner(
* @return An Optional containing a Future representing the pending scan task if the task is queued.
* The optional is empty if the task is not queued due to an ongoing scan.
*/
- public Optional<Future<?>> scanContainer(Container<?> container) {
- return scanContainer(container, scannerHelper);
+ public Optional<Future<?>> scanContainer(Container<?> container, String reasonForScan) {
+ return scanContainer(container, scannerHelper, reasonForScan);
}
/**
@@ -72,12 +74,12 @@ public Optional<Future<?>> scanContainer(Container<?> container) {
* @return An Optional containing a Future representing the pending scan task if the task is queued.
* The optional is empty if the task is not queued due to an ongoing scan.
*/
- public Optional<Future<?>> scanContainerWithoutGap(Container<?> container) {
- return scanContainer(container, scannerHelperWithoutGap);
+ public Optional<Future<?>> scanContainerWithoutGap(Container<?> container, String reasonForScan) {
+ return scanContainer(container, scannerHelperWithoutGap, reasonForScan);
}
- private Optional<Future<?>> scanContainer(Container<?> container, ContainerScanHelper helper) {
- if (!helper.shouldScanData(container)) {
+ private Optional<Future<?>> scanContainer(Container<?> container, ContainerScanHelper helper, String reasonForScan) {
+ if (!helper.shouldScanMetadata(container)) {
return Optional.empty();
}
@@ -85,9 +87,14 @@ private Optional<Future<?>> scanContainer(Container<?> container, ContainerScanH
long containerId = container.getContainerData().getContainerID();
if (addContainerToScheduledContainers(containerId)) {
resultFuture = scanExecutor.submit(() -> {
- performOnDemandScan(container, helper);
+ performOnDemandScan(container, helper, reasonForScan);
removeContainerFromScheduledContainers(containerId);
});
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping OnDemandScan for Container {} triggered due to '{}'. Reason: Already scheduled.",
+ containerId, reasonForScan);
+ }
}
return Optional.ofNullable(resultFuture);
}
@@ -101,9 +108,19 @@ private void removeContainerFromScheduledContainers(
containerRescheduleCheckSet.remove(containerId);
}
- private void performOnDemandScan(Container<?> container, ContainerScanHelper helper) {
+ private void performOnDemandScan(Container<?> container, ContainerScanHelper helper, String reasonForScan) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scheduling OnDemandScan for Container {}, Reason: {}",
+ container.getContainerData().getContainerID(), reasonForScan);
+ }
try {
- helper.scanData(container, throttler, canceler);
+ if (helper.shouldScanData(container)) {
+ helper.scanData(container, throttler, canceler);
+ } else {
+ // for containers that qualify for metadata scan and not data scan,
+ // like OPEN containers, trigger a metadata-only scan
+ helper.scanMetadata(container);
+ }
} catch (IOException e) {
LOG.warn("Unexpected exception while scanning container "
+ container.getContainerData().getContainerID(), e);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 9000be01ea3..db6c5352264 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -131,7 +131,7 @@ public class OzoneContainer {
private final XceiverServerSpi readChannel;
private final ContainerController controller;
private BackgroundContainerMetadataScanner metadataScanner;
- private OnDemandContainerDataScanner onDemandScanner;
+ private OnDemandContainerScanner onDemandScanner;
private List<BackgroundContainerDataScanner> dataScanners;
private List<AbstractBackgroundContainerScanner> backgroundScanners;
private final BlockDeletingService blockDeletingService;
@@ -205,8 +205,7 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService,
OZONE_RECOVERING_CONTAINER_TIMEOUT,
OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
this.witnessedContainerMetadataStore = WitnessedContainerMetadataStoreImpl.get(conf);
- containerSet = ContainerSet.newRwContainerSet(witnessedContainerMetadataStore.getContainerIdsTable(),
- recoveringContainerTimeout);
+ containerSet = ContainerSet.newRwContainerSet(witnessedContainerMetadataStore, recoveringContainerTimeout);
volumeSet.setGatherContainerUsages(this::gatherContainerUsages);
metadataScanner = null;
@@ -373,7 +372,8 @@ public void buildContainerSet() throws IOException {
for (Thread volumeThread : volumeThreads) {
volumeThread.join();
}
- try (TableIterator<ContainerID, ContainerID> itr = containerSet.getContainerIdsTable().keyIterator()) {
+ try (TableIterator<ContainerID, ContainerID> itr
+ = getWitnessedContainerMetadataStore().getContainerCreateInfoTable().keyIterator()) {
final Map<ContainerID, Long> containerIds = new HashMap<>();
while (itr.hasNext()) {
containerIds.put(itr.next(), 0L);
@@ -461,7 +461,7 @@ private void initOnDemandContainerScanner(ContainerScannerConfiguration c) {
"so the on-demand container data scanner will not start.");
return;
}
- onDemandScanner = new OnDemandContainerDataScanner(c, controller);
+ onDemandScanner = new OnDemandContainerScanner(c, controller);
containerSet.registerOnDemandScanner(onDemandScanner);
}
@@ -715,6 +715,10 @@ public void compactDb() {
}
}
+ public WitnessedContainerMetadataStore getWitnessedContainerMetadataStore() {
+ return witnessedContainerMetadataStore;
+ }
+
public DiskBalancerReportProto getDiskBalancerReport() {
if (diskBalancerService == null) {
return null;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
index a21fca4cc73..0786ccf0c24 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
@@ -116,13 +116,16 @@ public void importContainer(long containerID, Path tarFilePath,
}
ContainerUtils.verifyContainerFileChecksum(containerData, conf);
containerData.setVolume(targetVolume);
+ // lastDataScanTime should be cleared for an imported container
+ containerData.setDataScanTimestamp(null);
try (InputStream input = Files.newInputStream(tarFilePath)) {
Container container = controller.importContainer(
containerData, input, packer);
- // After container import is successful, increase used space for the volume
+ // After container import is successful, increase used space for the volume and schedule an OnDemand scan for it
targetVolume.incrementUsedSpace(container.getContainerData().getBytesUsed());
containerSet.addContainerByOverwriteMissingContainer(container);
+ containerSet.scanContainer(containerID, "Imported container");
}
} finally {
importContainerProgress.remove(containerID);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
new file mode 100644
index 00000000000..ed037070d80
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.upgrade;
+
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE;
+import static org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.ON_FINALIZE;
+import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
+
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.CodecException;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.metadata.ContainerCreateInfo;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerDBDefinition;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStoreImpl;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Upgrade Action for DataNode for update the table schema data of containerIds Table.
+ */
+@UpgradeActionHdds(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE, component = DATANODE, type = ON_FINALIZE)
+public class ContainerTableSchemaFinalizeAction
+ implements HDDSUpgradeAction<DatanodeStateMachine> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerTableSchemaFinalizeAction.class);
+
+ @Override
+ public void execute(DatanodeStateMachine arg) throws Exception {
+ WitnessedContainerMetadataStore metadataStore = arg.getContainer().getWitnessedContainerMetadataStore();
+ Table<ContainerID, ContainerCreateInfo> previousTable
+ = ((WitnessedContainerMetadataStoreImpl) metadataStore).getPreviousVersionTables().getContainerIdsTable();
+ Table<ContainerID, ContainerCreateInfo> currTable =
+ WitnessedContainerDBDefinition.CONTAINER_CREATE_INFO_TABLE_DEF.getTable(metadataStore.getStore());
+
+ // data is moved from old table to new table, no need cleanup if previous exist as this is just overwrite of data
+ try (TableIterator<ContainerID, ContainerID> curTblItr = currTable.keyIterator()) {
+ truncateCurrentTable(curTblItr, currTable);
+ }
+
+ try (BatchOperation batch = metadataStore.getStore().initBatchOperation();
+ TableIterator<ContainerID, ? extends Table.KeyValue<ContainerID, ContainerCreateInfo>> iterator =
+ previousTable.iterator()) {
+ while (iterator.hasNext()) {
+ Table.KeyValue<ContainerID, ContainerCreateInfo> next = iterator.next();
+ currTable.putWithBatch(batch, next.getKey(), next.getValue());
+ }
+ metadataStore.getStore().commitBatchOperation(batch);
+ LOG.info("Finished copy to containerIdsTable from previous table");
+ }
+ }
+
+ private static void truncateCurrentTable(
+ TableIterator<ContainerID, ContainerID> curTblItr,
+ Table<ContainerID, ContainerCreateInfo> currTable) throws RocksDatabaseException, CodecException {
+ // delete all previous entry if present in current table, this might come if previous upgrade is not finalized
+ // and crashed in between. Below logic for deleteRange is used to avoid tombstone creation for each entry
+ ContainerID startContainerID = null;
+ ContainerID endContainerID = null;
+ if (curTblItr.hasNext()) {
+ startContainerID = curTblItr.next();
+ endContainerID = startContainerID;
+ }
+ while (curTblItr.hasNext()) {
+ endContainerID = curTblItr.next();
+ }
+ if (startContainerID != null) {
+ if (startContainerID != endContainerID) {
+ currTable.deleteRange(startContainerID, endContainerID);
+ }
+ // remove last one entry as its exclusive end of range
+ currTable.delete(endContainerID);
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
index 681e302cede..f7879a33645 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
@@ -94,12 +94,11 @@ public boolean equals(Object o) {
return false;
}
ReconcileContainerCommand that = (ReconcileContainerCommand) o;
- return getContainerID() == that.getContainerID() &&
- Objects.equals(peerDatanodes, that.peerDatanodes);
+ return getContainerID() == that.getContainerID();
}
@Override
public int hashCode() {
- return Objects.hash(getContainerID(), peerDatanodes);
+ return Objects.hash(getContainerID());
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
index 63157450a19..953d3df16ce 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
@@ -145,6 +146,7 @@ public void testDeletedContainersClearedOnShutdown(String schemaVersion)
service.stop();
service.join();
service.close();
+ DefaultMetricsSystem.shutdown();
deletedContainersAfterShutdown =
hddsVolume.getDeletedContainerDir().listFiles();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
index 730c2c8b658..0f7794bac4c 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.checksum;
import static org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager.getContainerChecksumFile;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -36,6 +37,7 @@
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
@@ -43,6 +45,9 @@
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
@@ -161,7 +166,7 @@ public static ContainerMerkleTreeWriter buildTestTree(ConfigurationSource conf,
int numCorruptChunks) {
ContainerProtos.ContainerMerkleTree.Builder treeBuilder = originalTree.toProto().toBuilder();
- ContainerDiffReport diff = new ContainerDiffReport();
+ ContainerDiffReport diff = new ContainerDiffReport(1);
introduceMissingBlocks(treeBuilder, numMissingBlocks, diff);
introduceMissingChunks(treeBuilder, numMissingChunks, diff);
@@ -323,12 +328,12 @@ private static void assertEqualsChunkMerkleTree(List<ContainerProtos.ChunkMerkle
}
/**
- * This function checks whether the container checksum file exists.
+ * This function checks whether the container checksum file exists for a container in a given datanode.
*/
public static boolean containerChecksumFileExists(HddsDatanodeService hddsDatanode, long containerID) {
OzoneContainer ozoneContainer = hddsDatanode.getDatanodeStateMachine().getContainer();
Container<?> container = ozoneContainer.getController().getContainer(containerID);
- return ContainerChecksumTreeManager.checksumFileExist(container);
+ return getContainerChecksumFile(container.getContainerData()).exists();
}
public static void writeContainerDataTreeProto(ContainerData data, ContainerProtos.ContainerMerkleTree tree)
@@ -346,4 +351,38 @@ public static void writeContainerDataTreeProto(ContainerData data, ContainerProt
}
data.setDataChecksum(checksumInfo.getContainerMerkleTree().getDataChecksum());
}
+
+ /**
+ * This function verifies that the in-memory data checksum matches the one stored in the container data and
+ * the RocksDB.
+ *
+ * @param containerData The container data to verify.
+ * @param conf The Ozone configuration.
+ * @throws IOException If an error occurs while reading the checksum info or RocksDB.
+ */
+ public static void verifyAllDataChecksumsMatch(KeyValueContainerData containerData, OzoneConfiguration conf)
+ throws IOException {
+ assertNotNull(containerData, "Container data should not be null");
+ ContainerProtos.ContainerChecksumInfo containerChecksumInfo = ContainerChecksumTreeManager
+ .readChecksumInfo(containerData);
+ assertNotNull(containerChecksumInfo);
+ long dataChecksum = containerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+ Long dbDataChecksum;
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+ dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+ }
+
+ if (containerData.getDataChecksum() == 0) {
+ assertEquals(containerData.getDataChecksum(), dataChecksum);
+ // RocksDB checksum can be null if the file doesn't exist or when the file is created by
+ // the block deleting service. 0 checksum will be stored when the container is loaded without
+ // merkle tree.
+ assertThat(dbDataChecksum).isIn(0L, null);
+ } else {
+ // In-Memory, Container Merkle Tree file, RocksDB checksum should be equal
+ assertEquals(containerData.getDataChecksum(), dataChecksum, "In-memory data checksum should match " +
+ "the one in the checksum file.");
+ assertEquals(dbDataChecksum, dataChecksum);
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java
index 36b705c9926..65478afa303 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java
@@ -39,7 +39,6 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
@@ -98,6 +97,8 @@ public void init() {
container = mock(KeyValueContainerData.class);
when(container.getContainerID()).thenReturn(CONTAINER_ID);
when(container.getMetadataPath()).thenReturn(testDir.getAbsolutePath());
+ // File name is hardcoded here to check if the file name has been changed, since this would
+ // need additional compatibility handling.
checksumFile = new File(testDir, CONTAINER_ID + ".tree");
checksumManager = new ContainerChecksumTreeManager(new OzoneConfiguration());
config = new OzoneConfiguration();
@@ -344,8 +345,8 @@ public void testContainerWithNoDiff() throws Exception {
ContainerProtos.ContainerChecksumInfo peerChecksumInfo = ContainerProtos.ContainerChecksumInfo.newBuilder()
.setContainerID(container.getContainerID())
.setContainerMerkleTree(peerMerkleTree.toProto()).build();
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport diff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport diff = checksumManager.diff(checksumInfo, peerChecksumInfo);
assertTrue(checksumManager.getMetrics().getMerkleTreeDiffLatencyNS().lastStat().total() > 0);
assertFalse(diff.needsRepair());
assertEquals(checksumManager.getMetrics().getNoRepairContainerDiffs(), 1);
@@ -368,8 +369,8 @@ public void testContainerDiffWithMismatches(int numMissingBlock, int numMissingC
ContainerProtos.ContainerChecksumInfo peerChecksumInfo = ContainerProtos.ContainerChecksumInfo.newBuilder()
.setContainerID(container.getContainerID())
.setContainerMerkleTree(peerMerkleTree.toProto()).build();
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport diff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport diff = checksumManager.diff(checksumInfo, peerChecksumInfo);
assertTrue(checksumManager.getMetrics().getMerkleTreeDiffLatencyNS().lastStat().total() > 0);
assertContainerDiffMatch(expectedDiff, diff);
assertEquals(1, checksumManager.getMetrics().getRepairContainerDiffs());
@@ -395,8 +396,8 @@ public void testPeerWithMismatchesHasNoDiff(int numMissingBlock, int numMissingC
ContainerProtos.ContainerChecksumInfo peerChecksumInfo = ContainerProtos.ContainerChecksumInfo.newBuilder()
.setContainerID(container.getContainerID())
.setContainerMerkleTree(peerMerkleTree).build();
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport diff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport diff = checksumManager.diff(checksumInfo, peerChecksumInfo);
assertFalse(diff.needsRepair());
assertEquals(checksumManager.getMetrics().getNoRepairContainerDiffs(), 1);
assertEquals(0, checksumManager.getMetrics().getMissingBlocksIdentified());
@@ -409,8 +410,8 @@ public void testFailureContainerMerkleTreeMetric() throws IOException {
ContainerProtos.ContainerChecksumInfo peerChecksum = ContainerProtos.ContainerChecksumInfo.newBuilder().build();
ContainerMerkleTreeWriter ourMerkleTree = buildTestTree(config);
checksumManager.writeContainerDataTree(container, ourMerkleTree);
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- assertThrows(StorageContainerException.class, () -> checksumManager.diff(checksumInfo.get(), peerChecksum));
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ assertThrows(StorageContainerException.class, () -> checksumManager.diff(checksumInfo, peerChecksum));
assertEquals(checksumManager.getMetrics().getMerkleTreeDiffFailure(), 1);
}
@@ -436,8 +437,8 @@ void testDeletedBlocksInPeerAndBoth() throws Exception {
.addAllDeletedBlocks(deletedBlockList).build();
writeContainerDataTreeProto(container, ourMerkleTree);
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
// The diff should not have any missing block/missing chunk/corrupt chunks as the blocks are deleted
// in peer merkle tree.
@@ -448,7 +449,7 @@ void testDeletedBlocksInPeerAndBoth() throws Exception {
// Delete blocks in our merkle tree as well.
checksumManager.markBlocksAsDeleted(container, blockIDs);
checksumInfo = checksumManager.read(container);
- containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
// The diff should not have any missing block/missing chunk/corrupt chunks as the blocks are deleted
// in both merkle tree.
@@ -474,8 +475,8 @@ void testDeletedBlocksInOurContainerOnly() throws Exception {
writeContainerDataTreeProto(container, ourMerkleTree);
checksumManager.markBlocksAsDeleted(container, deletedBlockList);
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
// The diff should not have any missing block/missing chunk/corrupt chunks as the blocks are deleted
// in our merkle tree.
@@ -507,8 +508,8 @@ void testCorruptionInOurMerkleTreeAndDeletedBlocksInPeer() throws Exception {
writeContainerDataTreeProto(container, ourMerkleTree);
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
// The diff should not have any missing block/missing chunk/corrupt chunks as the blocks are deleted
// in peer merkle tree.
@@ -539,8 +540,8 @@ void testContainerDiffWithBlockDeletionInPeer() throws Exception {
writeContainerDataTreeProto(container, ourMerkleTree);
ContainerProtos.ContainerChecksumInfo peerChecksumInfo = peerChecksumInfoBuilder.build();
- Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = checksumManager.read(container);
- ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(container);
+ ContainerDiffReport containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
// The diff should not have any missing block/missing chunk/corrupt chunks as the blocks are deleted
// in peer merkle tree.
assertFalse(containerDiff.getMissingBlocks().isEmpty());
@@ -553,7 +554,7 @@ void testContainerDiffWithBlockDeletionInPeer() throws Exception {
// Clear deleted blocks to add them in missing blocks.
peerChecksumInfo = peerChecksumInfoBuilder.clearDeletedBlocks().build();
checksumInfo = checksumManager.read(container);
- containerDiff = checksumManager.diff(checksumInfo.get(), peerChecksumInfo);
+ containerDiff = checksumManager.diff(checksumInfo, peerChecksumInfo);
assertFalse(containerDiff.getMissingBlocks().isEmpty());
// Missing block does not contain the deleted blocks 6L to 10L
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
index 2eaad40dc5f..a3069997387 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
@@ -113,6 +113,103 @@ public void testBuildTreeWithMissingChunks() {
assertTreesSortedAndMatch(expectedTree, actualTreeProto);
}
+ @Test
+ public void testBlockIdIncludedInChecksum() {
+ // Create a set of chunks to be used in different blocks with identical content.
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0, ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1, ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Create two blocks with different IDs but identical chunk data
+ final long blockID1 = 1;
+ final long blockID2 = 2;
+
+ ContainerMerkleTreeWriter tree1 = new ContainerMerkleTreeWriter();
+ tree1.addChunks(blockID1, true, chunk1, chunk2);
+
+ ContainerMerkleTreeWriter tree2 = new ContainerMerkleTreeWriter();
+ tree2.addChunks(blockID2, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree tree1Proto = tree1.toProto();
+ ContainerProtos.ContainerMerkleTree tree2Proto = tree2.toProto();
+
+ // Even though the chunks are identical, the block checksums should be different
+ // because the block IDs are different
+ ContainerProtos.BlockMerkleTree block1 = tree1Proto.getBlockMerkleTree(0);
+ ContainerProtos.BlockMerkleTree block2 = tree2Proto.getBlockMerkleTree(0);
+
+ assertEquals(blockID1, block1.getBlockID());
+ assertEquals(blockID2, block2.getBlockID());
+ assertNotEquals(block1.getDataChecksum(), block2.getDataChecksum(),
+ "Blocks with identical chunks but different IDs should have different checksums");
+
+ // Consequently, the container checksums should also be different
+ assertNotEquals(tree1Proto.getDataChecksum(), tree2Proto.getDataChecksum(),
+ "Containers with blocks having identical chunks but different IDs should have different checksums");
+ }
+
+ @Test
+ public void testIdenticalBlocksHaveSameChecksum() {
+ // Create a set of chunks to be used in different blocks with identical content.
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0, ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1, ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Create two blocks with the same ID and identical chunk data
+ final long blockID = 1;
+
+ ContainerMerkleTreeWriter tree1 = new ContainerMerkleTreeWriter();
+ tree1.addChunks(blockID, true, chunk1, chunk2);
+
+ ContainerMerkleTreeWriter tree2 = new ContainerMerkleTreeWriter();
+ tree2.addChunks(blockID, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree tree1Proto = tree1.toProto();
+ ContainerProtos.ContainerMerkleTree tree2Proto = tree2.toProto();
+
+ // Blocks with same ID and identical chunks should have same checksums
+ ContainerProtos.BlockMerkleTree block1 = tree1Proto.getBlockMerkleTree(0);
+ ContainerProtos.BlockMerkleTree block2 = tree2Proto.getBlockMerkleTree(0);
+
+ assertEquals(blockID, block1.getBlockID());
+ assertEquals(blockID, block2.getBlockID());
+ assertEquals(block1.getDataChecksum(), block2.getDataChecksum(),
+ "Blocks with same ID and identical chunks should have same checksums");
+
+ // Container checksums should also be the same
+ assertEquals(tree1Proto.getDataChecksum(), tree2Proto.getDataChecksum(),
+ "Containers with identical blocks should have same checksums");
+ }
+
+ @Test
+ public void testContainerReplicasWithDifferentMissingBlocksHaveDifferentChecksums() {
+ // Create identical chunk data that will be used across all blocks
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0, ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1, ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Scenario: Container has 5 identical blocks, but different replicas are missing different blocks
+ // Replica 1 is missing block 1 (has blocks 2,3,4,5)
+ ContainerMerkleTreeWriter replica1 = new ContainerMerkleTreeWriter();
+ replica1.addChunks(2, true, chunk1, chunk2);
+ replica1.addChunks(3, true, chunk1, chunk2);
+ replica1.addChunks(4, true, chunk1, chunk2);
+ replica1.addChunks(5, true, chunk1, chunk2);
+
+ // Replica 2 is missing block 5 (has blocks 1,2,3,4)
+ ContainerMerkleTreeWriter replica2 = new ContainerMerkleTreeWriter();
+ replica2.addChunks(1, true, chunk1, chunk2);
+ replica2.addChunks(2, true, chunk1, chunk2);
+ replica2.addChunks(3, true, chunk1, chunk2);
+ replica2.addChunks(4, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree replica1Proto = replica1.toProto();
+ ContainerProtos.ContainerMerkleTree replica2Proto = replica2.toProto();
+ assertNotEquals(replica1Proto.getDataChecksum(), replica2Proto.getDataChecksum(),
+ "Container replicas with identical blocks but different missing blocks should have different checksums");
+
+ // Verify both replicas have the same number of blocks
+ assertEquals(4, replica1Proto.getBlockMerkleTreeCount());
+ assertEquals(4, replica2Proto.getBlockMerkleTreeCount());
+ }
+
@Test
public void testBuildTreeWithEmptyBlock() {
final long blockID = 1;
@@ -274,12 +371,12 @@ private ContainerProtos.ContainerMerkleTree buildExpectedContainerTree(List<Cont
private ContainerProtos.BlockMerkleTree buildExpectedBlockTree(long blockID,
List<ContainerProtos.ChunkMerkleTree> chunks) {
+ List<Long> itemsToChecksum = chunks.stream().map(ContainerProtos.ChunkMerkleTree::getDataChecksum)
+ .collect(Collectors.toList());
+ itemsToChecksum.add(0, blockID);
return ContainerProtos.BlockMerkleTree.newBuilder()
.setBlockID(blockID)
- .setDataChecksum(computeExpectedChecksum(
- chunks.stream()
- .map(ContainerProtos.ChunkMerkleTree::getDataChecksum)
- .collect(Collectors.toList())))
+ .setDataChecksum(computeExpectedChecksum(itemsToChecksum))
.addAllChunkMerkleTree(chunks)
.build();
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestReconcileContainerTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestReconcileContainerTask.java
index 3a5826c21ed..28ff135fa26 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestReconcileContainerTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestReconcileContainerTask.java
@@ -88,8 +88,8 @@ public void testEqualityWhenContainerIDsMatch() {
// Same container ID and peers.
assertEquals(peerSet1Task, otherPeerSet1Task);
- // Same container ID, different peers.
- assertNotEquals(peerSet1Task, peerSet2Task);
+ // Same container ID, different peers - should still be equal now.
+ assertEquals(peerSet1Task, peerSet2Task);
}
@Test
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 6af2e00ade5..5d674de7fcb 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.ozone.container.common;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import java.util.UUID;
@@ -116,4 +119,30 @@ public void testKeyValueData(ContainerTestVersionInfo versionInfo) {
assertEquals(kvData.getSchemaVersion(), newKvData.getSchemaVersion());
}
+ @ContainerTestVersionInfo.ContainerTest
+ public void testNeedsDataChecksum(ContainerTestVersionInfo versionInfo) {
+ initVersionInfo(versionInfo);
+
+ KeyValueContainerData containerData = new KeyValueContainerData(1, layout, MAXSIZE, UUID.randomUUID().toString(),
+ UUID.randomUUID().toString());
+
+ // When the container is initially created without a checksum, the checksum will be 0 but the container still
+ // indicates it needs the actual one generated.
+ assertFalse(containerData.isEmpty());
+ assertTrue(containerData.needsDataChecksum());
+ assertEquals(0, containerData.getDataChecksum());
+
+ // Once the setter is called with any value, the container should no longer consider the checksum missing.
+ containerData.setDataChecksum(0);
+ assertFalse(containerData.needsDataChecksum());
+ assertEquals(0, containerData.getDataChecksum());
+
+ containerData.setDataChecksum(123L);
+ assertFalse(containerData.isEmpty());
+ assertFalse(containerData.needsDataChecksum());
+ assertEquals(123L, containerData.getDataChecksum());
+
+ assertThrows(IllegalArgumentException.class, () -> containerData.setDataChecksum(-1L),
+ "Negative checksum value should throw an exception.");
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/ContainerImplTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/ContainerImplTestUtils.java
index 1e27e748d69..e005c426333 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/ContainerImplTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/ContainerImplTestUtils.java
@@ -17,8 +17,12 @@
package org.apache.hadoop.ozone.container.common.impl;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.time.Clock;
import org.apache.hadoop.hdds.utils.db.InMemoryTestTable;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
/**
* Helper utility to test container impl.
@@ -33,10 +37,14 @@ public static ContainerSet newContainerSet() {
}
public static ContainerSet newContainerSet(long recoveringTimeout) {
- return ContainerSet.newRwContainerSet(new InMemoryTestTable<>(), recoveringTimeout);
+ WitnessedContainerMetadataStore mockMetadataStore = mock(WitnessedContainerMetadataStore.class);
+ when(mockMetadataStore.getContainerCreateInfoTable()).thenReturn(new InMemoryTestTable<>());
+ return ContainerSet.newRwContainerSet(mockMetadataStore, recoveringTimeout);
}
public static ContainerSet newContainerSet(long recoveringTimeout, Clock clock) {
- return new ContainerSet(new InMemoryTestTable<>(), recoveringTimeout, clock);
+ WitnessedContainerMetadataStore mockMetadataStore = mock(WitnessedContainerMetadataStore.class);
+ when(mockMetadataStore.getContainerCreateInfoTable()).thenReturn(new InMemoryTestTable<>());
+ return new ContainerSet(mockMetadataStore, recoveringTimeout, clock);
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index 10c897b1f54..8c54dd848af 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -27,6 +27,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -51,7 +52,7 @@
import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
/**
* Class used to test ContainerSet operations.
@@ -62,6 +63,8 @@ public class TestContainerSet {
private ContainerLayoutVersion layoutVersion;
+ private static final String TEST_SCAN = "Test Scan";
+
private void setLayoutVersion(ContainerLayoutVersion layoutVersion) {
this.layoutVersion = layoutVersion;
}
@@ -293,11 +296,11 @@ public void testContainerScanHandler(ContainerLayoutVersion layout) throws Excep
setLayoutVersion(layout);
ContainerSet containerSet = createContainerSet();
// Scan when no handler is registered should not throw an exception.
- containerSet.scanContainer(FIRST_ID);
+ containerSet.scanContainer(FIRST_ID, TEST_SCAN);
AtomicLong invocationCount = new AtomicLong();
- OnDemandContainerDataScanner mockScanner = mock(OnDemandContainerDataScanner.class);
- when(mockScanner.scanContainer(any())).then(inv -> {
+ OnDemandContainerScanner mockScanner = mock(OnDemandContainerScanner.class);
+ when(mockScanner.scanContainer(any(), anyString())).then(inv -> {
KeyValueContainer c = inv.getArgument(0);
// If the handler was incorrectly triggered for a non-existent container, this assert would fail.
assertEquals(FIRST_ID, c.getContainerData().getContainerID());
@@ -307,11 +310,11 @@ public void testContainerScanHandler(ContainerLayoutVersion layout) throws Excep
containerSet.registerOnDemandScanner(mockScanner);
// Scan of an existing container when a handler is registered should trigger a scan.
- containerSet.scanContainer(FIRST_ID);
+ containerSet.scanContainer(FIRST_ID, TEST_SCAN);
assertEquals(1, invocationCount.get());
// Scan of non-existent container should not throw exception or trigger an additional invocation.
- containerSet.scanContainer(FIRST_ID - 1);
+ containerSet.scanContainer(FIRST_ID - 1, TEST_SCAN);
assertEquals(1, invocationCount.get());
}
@@ -320,11 +323,11 @@ public void testContainerScanHandlerWithoutGap(ContainerLayoutVersion layout) th
setLayoutVersion(layout);
ContainerSet containerSet = createContainerSet();
// Scan when no handler is registered should not throw an exception.
- containerSet.scanContainer(FIRST_ID);
+ containerSet.scanContainer(FIRST_ID, TEST_SCAN);
AtomicLong invocationCount = new AtomicLong();
- OnDemandContainerDataScanner mockScanner = mock(OnDemandContainerDataScanner.class);
- when(mockScanner.scanContainerWithoutGap(any())).then(inv -> {
+ OnDemandContainerScanner mockScanner = mock(OnDemandContainerScanner.class);
+ when(mockScanner.scanContainerWithoutGap(any(), anyString())).then(inv -> {
KeyValueContainer c = inv.getArgument(0);
// If the handler was incorrectly triggered for a non-existent container, this assert would fail.
assertEquals(FIRST_ID, c.getContainerData().getContainerID());
@@ -334,11 +337,11 @@ public void testContainerScanHandlerWithoutGap(ContainerLayoutVersion layout) th
containerSet.registerOnDemandScanner(mockScanner);
// Scan of an existing container when a handler is registered should trigger a scan.
- containerSet.scanContainerWithoutGap(FIRST_ID);
+ containerSet.scanContainerWithoutGap(FIRST_ID, TEST_SCAN);
assertEquals(1, invocationCount.get());
// Scan of non-existent container should not throw exception or trigger an additional invocation.
- containerSet.scanContainerWithoutGap(FIRST_ID - 1);
+ containerSet.scanContainerWithoutGap(FIRST_ID - 1, TEST_SCAN);
assertEquals(1, invocationCount.get());
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
index bc9c33197a0..85d20f509af 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteBlocksCommandHandler.java
@@ -40,13 +40,16 @@
import java.io.IOException;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -271,6 +274,28 @@ public List<Future<DeleteBlockTransactionExecutionResult>> answer(
blockDeleteMetrics.getTotalLockTimeoutTransactionCount());
}
+ @Test
+ public void testDeleteBlocksCommandHandlerExceptionShouldNotInterrupt() throws Exception {
+ setup();
+ // future task will throw first execution exception, and next one will succeed
+ doAnswer((Answer<List<Future<DeleteBlockTransactionExecutionResult>>>) invocationOnMock -> {
+ List<Future<DeleteBlockTransactionExecutionResult>> result = new ArrayList<>();
+ CompletableFuture<DeleteBlockTransactionExecutionResult> future =
+ new CompletableFuture<>();
+ future.completeExceptionally(new ExecutionException("Simulated Exception", new IOException()));
+ result.add(future);
+ future = new CompletableFuture<>();
+ future.complete(new DeleteBlockTransactionExecutionResult(null, false));
+ result.add(future);
+ return result;
+ }).when(handler).submitTasks(any());
+
+ // last task as success should be returned as result, ignoring the first failed task
+ List<DeleteBlockTransactionResult> deleteBlockTransactionResults =
+ handler.executeCmdWithRetry(Collections.emptyList());
+ assertEquals(1, deleteBlockTransactionResults.size());
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void testDeleteCmdWorkerInterval(
ContainerTestVersionInfo versionInfo) throws Exception {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
similarity index 75%
rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java
rename to hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
index 2f166bea022..170e20abca8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
@@ -15,13 +15,5 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.om.snapshot;
-
-/**
- * OmSnapshot file system tests for FSO.
- */
-public class TestOmSnapshotFileSystemFso extends TestOmSnapshotFileSystem {
- TestOmSnapshotFileSystemFso() throws Exception {
- super(BUCKET_NAME_FSO, false);
- }
-}
+/** Tests for Common container utils. */
+package org.apache.hadoop.ozone.container.common.utils;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index d36191d03da..0bd576732a0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -532,12 +532,14 @@ public void testDBDirFailureDetected() throws Exception {
volume.createTmpDirs(CLUSTER_ID);
VolumeCheckResult result = volume.check(false);
+ assertEquals(1, volume.getVolumeInfoStats().getNumScans());
assertEquals(VolumeCheckResult.HEALTHY, result);
File dbFile = new File(volume.getDbParentDir(), CONTAINER_DB_NAME);
FileUtils.deleteDirectory(dbFile);
result = volume.check(false);
+ assertEquals(2, volume.getVolumeInfoStats().getNumScans());
assertEquals(VolumeCheckResult.FAILED, result);
volume.shutdown();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java
index 7ffc73f0868..2cdd06fa51b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java
@@ -25,6 +25,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -43,13 +44,14 @@
*/
public class TestPeriodicVolumeChecker {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestPeriodicVolumeChecker.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestPeriodicVolumeChecker.class);
@TempDir
private Path folder;
private OzoneConfiguration conf = new OzoneConfiguration();
+ private MutableVolumeSet volumeSet;
+ private MutableVolumeSet metaVolumeSet;
@BeforeEach
public void setup() throws IOException {
@@ -57,58 +59,69 @@ public void setup() throws IOException {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, folder.toString());
conf.set(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR,
Files.createDirectory(folder.resolve("VolumeCheckerDir")).toString());
+ String scmId = UUID.randomUUID().toString();
+ String datanodeUuid = UUID.randomUUID().toString();
+ volumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ metaVolumeSet = new MutableVolumeSet(datanodeUuid, scmId, conf, null,
+ StorageVolume.VolumeType.META_VOLUME, null);
}
@Test
public void testPeriodicVolumeChecker(TestInfo testInfo) throws Exception {
LOG.info("Executing {}", testInfo.getTestMethod());
- DatanodeConfiguration dnConf =
- conf.getObject(DatanodeConfiguration.class);
+ DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
Duration gap = dnConf.getDiskCheckMinGap();
- Duration interval = Duration.ofMinutes(
- dnConf.getPeriodicDiskCheckIntervalMinutes());
+ Duration interval = Duration.ofMinutes(dnConf.getPeriodicDiskCheckIntervalMinutes());
FakeTimer timer = new FakeTimer();
- StorageVolumeChecker volumeChecker = new StorageVolumeChecker(conf, timer,
- "");
+ StorageVolumeChecker volumeChecker = new StorageVolumeChecker(conf, timer, "");
+ StorageVolumeScannerMetrics metrics = volumeChecker.getMetrics();
try {
- volumeChecker.registerVolumeSet(new ImmutableVolumeSet(makeVolumes(
- 2, HEALTHY)));
- volumeChecker.registerVolumeSet(new ImmutableVolumeSet(makeVolumes(
- 1, HEALTHY)));
- volumeChecker.setDelegateChecker(
- new TestStorageVolumeChecker.DummyChecker());
+ volumeChecker.registerVolumeSet(new ImmutableVolumeSet(makeVolumes(2, HEALTHY)));
+ volumeChecker.registerVolumeSet(new ImmutableVolumeSet(makeVolumes(1, HEALTHY)));
+ volumeChecker.registerVolumeSet(volumeSet);
+ volumeChecker.registerVolumeSet(metaVolumeSet);
+ volumeChecker.setDelegateChecker(new TestStorageVolumeChecker.DummyChecker());
- assertEquals(0, volumeChecker.getNumAllVolumeChecks());
- assertEquals(0, volumeChecker.getNumAllVolumeSetsChecks());
+ assertEquals(0, metrics.getNumScanIterations());
+ assertEquals(0, metrics.getNumDataVolumeScans());
+ assertEquals(0, metrics.getNumMetadataVolumeScans());
+ assertEquals(0, metrics.getNumVolumesScannedInLastIteration());
// first round
timer.advance(gap.toMillis() / 3);
volumeChecker.checkAllVolumeSets();
- assertEquals(2, volumeChecker.getNumAllVolumeChecks());
- assertEquals(1, volumeChecker.getNumAllVolumeSetsChecks());
- assertEquals(0, volumeChecker.getNumSkippedChecks());
+ assertEquals(1, metrics.getNumScanIterations());
+ assertEquals(1, metrics.getNumDataVolumeScans());
+ assertEquals(1, metrics.getNumMetadataVolumeScans());
+ assertEquals(5, metrics.getNumVolumesScannedInLastIteration());
+ assertEquals(0, metrics.getNumIterationsSkipped());
// periodic disk checker next round within gap
timer.advance(gap.toMillis() / 3);
volumeChecker.checkAllVolumeSets();
// skipped next round
- assertEquals(2, volumeChecker.getNumAllVolumeChecks());
- assertEquals(1, volumeChecker.getNumAllVolumeSetsChecks());
- assertEquals(1, volumeChecker.getNumSkippedChecks());
+ assertEquals(1, metrics.getNumScanIterations());
+ assertEquals(1, metrics.getNumDataVolumeScans());
+ assertEquals(1, metrics.getNumMetadataVolumeScans());
+ assertEquals(5, metrics.getNumVolumesScannedInLastIteration());
+ assertEquals(1, metrics.getNumIterationsSkipped());
// periodic disk checker next round
timer.advance(interval.toMillis());
volumeChecker.checkAllVolumeSets();
- assertEquals(4, volumeChecker.getNumAllVolumeChecks());
- assertEquals(2, volumeChecker.getNumAllVolumeSetsChecks());
- assertEquals(1, volumeChecker.getNumSkippedChecks());
+ assertEquals(2, metrics.getNumScanIterations());
+ assertEquals(2, metrics.getNumDataVolumeScans());
+ assertEquals(2, metrics.getNumMetadataVolumeScans());
+ assertEquals(5, metrics.getNumVolumesScannedInLastIteration());
+ assertEquals(1, metrics.getNumIterationsSkipped());
} finally {
volumeChecker.shutdownAndWait(1, TimeUnit.SECONDS);
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
index 4fc1b19fa6a..92287bc617a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java
@@ -59,6 +59,7 @@
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.FakeTimer;
import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -252,6 +253,52 @@ public void testVolumeDeletion(VolumeCheckResult checkResult,
ozoneContainer.stop();
}
+ /**
+ * Test numScansSkipped metric from VolumeInfoMetrics when volume check is skipped.
+ */
+ @Test
+ public void testNumScansSkipped() throws Exception {
+ initTest(VolumeCheckResult.HEALTHY);
+
+ final List<HddsVolume> volumes = makeVolumes(3, expectedVolumeHealth);
+
+ FakeTimer timer = new FakeTimer();
+ final StorageVolumeChecker checker =
+ new StorageVolumeChecker(new OzoneConfiguration(), timer, "");
+
+ VolumeInfoMetrics metrics1 = new VolumeInfoMetrics("test-volume-1", volumes.get(0));
+ VolumeInfoMetrics metrics2 = new VolumeInfoMetrics("test-volume-2", volumes.get(1));
+ VolumeInfoMetrics metrics3 = new VolumeInfoMetrics("test-volume-3", volumes.get(2));
+ when(volumes.get(0).getVolumeInfoStats()).thenReturn(metrics1);
+ when(volumes.get(1).getVolumeInfoStats()).thenReturn(metrics2);
+ when(volumes.get(2).getVolumeInfoStats()).thenReturn(metrics3);
+
+ checker.checkAllVolumes(volumes);
+
+ // No volume skipped on first call
+ assertEquals(0, metrics1.getNumScansSkipped());
+ assertEquals(0, metrics2.getNumScansSkipped());
+ assertEquals(0, metrics3.getNumScansSkipped());
+
+ // Second call skipped because msSinceLastCheck (8 mins) < minMsBetweenChecks (10 mins)
+ timer.advance(480_000);
+ checker.checkAllVolumes(volumes);
+
+ assertEquals(1, metrics1.getNumScansSkipped());
+ assertEquals(1, metrics2.getNumScansSkipped());
+ assertEquals(1, metrics3.getNumScansSkipped());
+
+ // Third call should not skip volumes as msSinceLastCheck (11 mins) > minMsBetweenChecks (10 mins)
+ timer.advance(180_000);
+ checker.checkAllVolumes(volumes);
+
+ assertEquals(1, metrics1.getNumScansSkipped());
+ assertEquals(1, metrics2.getNumScansSkipped());
+ assertEquals(1, metrics3.getNumScansSkipped());
+
+ checker.shutdownAndWait(0, TimeUnit.SECONDS);
+ }
+
/**
* A checker to wraps the result of {@link HddsVolume#check} in
* an ImmediateFuture.
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
index e95fc3ab3ec..b90df135f00 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
@@ -27,6 +28,7 @@
import static org.assertj.core.api.Assertions.fail;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -81,7 +83,7 @@
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
-import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.jupiter.api.AfterAll;
@@ -121,6 +123,8 @@ public class TestContainerReconciliationWithMockDatanodes {
private static final int CHUNKS_PER_BLOCK = 4;
private static final int NUM_DATANODES = 3;
+ private static final String TEST_SCAN = "Test Scan";
+
/**
* Number of corrupt blocks and chunks.
*
@@ -159,12 +163,16 @@ public static void setup() throws Exception {
// Use this fake host name to track the node through the test since it's easier to visualize than a UUID.
dnDetails.setHostName("dn" + (i + 1));
MockDatanode dn = new MockDatanode(dnDetails, containerDir);
+ // This will close the container and build a data checksum based on the chunk checksums in the metadata.
dn.addContainerWithBlocks(CONTAINER_ID, 15);
datanodes.add(dn);
}
+ long dataChecksumFromMetadata = assertUniqueChecksumCount(CONTAINER_ID, datanodes, 1);
+ assertNotEquals(0, dataChecksumFromMetadata);
datanodes.forEach(d -> d.scanContainer(CONTAINER_ID));
healthyDataChecksum = assertUniqueChecksumCount(CONTAINER_ID, datanodes, 1);
+ assertEquals(dataChecksumFromMetadata, healthyDataChecksum);
// Do not count the initial synchronous scan to build the merkle tree towards the scan count in the tests.
// This lets each test run start counting the number of scans from zero.
datanodes.forEach(MockDatanode::resetOnDemandScanCount);
@@ -299,7 +307,7 @@ private static void mockContainerProtocolCalls() {
private static class MockDatanode {
private final KeyValueHandler handler;
private final DatanodeDetails dnDetails;
- private final OnDemandContainerDataScanner onDemandScanner;
+ private final OnDemandContainerScanner onDemandScanner;
private final ContainerSet containerSet;
private final OzoneConfiguration conf;
@@ -322,7 +330,7 @@ private static class MockDatanode {
ContainerController controller = new ContainerController(containerSet,
Collections.singletonMap(ContainerProtos.ContainerType.KeyValueContainer, handler));
- onDemandScanner = new OnDemandContainerDataScanner(
+ onDemandScanner = new OnDemandContainerScanner(
conf.getObject(ContainerScannerConfiguration.class), controller);
// Register the on-demand container scanner with the container set used by the KeyValueHandler.
containerSet.registerOnDemandScanner(onDemandScanner);
@@ -350,13 +358,13 @@ public ContainerProtos.GetContainerChecksumInfoResponseProto getChecksumInfo(lon
*/
public long checkAndGetDataChecksum(long containerID) {
KeyValueContainer container = getContainer(containerID);
+ KeyValueContainerData containerData = container.getContainerData();
long dataChecksum = 0;
try {
- Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
- handler.getChecksumManager().read(container.getContainerData());
- assertTrue(containerChecksumInfo.isPresent());
- dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
- assertEquals(container.getContainerData().getDataChecksum(), dataChecksum);
+ ContainerProtos.ContainerChecksumInfo containerChecksumInfo = handler.getChecksumManager()
+ .read(containerData);
+ dataChecksum = containerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+ verifyAllDataChecksumsMatch(containerData, conf);
} catch (IOException ex) {
fail("Failed to read container checksum from disk", ex);
}
@@ -420,7 +428,8 @@ public KeyValueContainer getContainer(long containerID) {
* Triggers a synchronous scan of the container. This method will block until the scan completes.
*/
public void scanContainer(long containerID) {
- Optional<Future<?>> scanFuture = onDemandScanner.scanContainerWithoutGap(containerSet.getContainer(containerID));
+ Optional<Future<?>> scanFuture = onDemandScanner.scanContainerWithoutGap(containerSet.getContainer(containerID),
+ TEST_SCAN);
assertTrue(scanFuture.isPresent());
try {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index 7ba72f937ef..28c118b517f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -20,6 +20,8 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V2;
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED;
import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
import static org.apache.hadoop.ozone.container.replication.CopyContainerCompression.NO_COMPRESSION;
@@ -73,6 +75,7 @@
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
@@ -291,6 +294,53 @@ public void testEmptyContainerImportExport(
checkContainerFilesPresent(data, 0);
}
+ @ContainerTestVersionInfo.ContainerTest
+ public void testEmptyMerkleTreeImportExport(ContainerTestVersionInfo versionInfo) throws Exception {
+ init(versionInfo);
+ createContainer();
+ closeContainer();
+
+ KeyValueContainerData data = keyValueContainer.getContainerData();
+ // Create an empty checksum file that exists but has no valid merkle tree
+ File checksumFile = ContainerChecksumTreeManager.getContainerChecksumFile(data);
+ ContainerProtos.ContainerChecksumInfo emptyContainerInfo = ContainerProtos.ContainerChecksumInfo
+ .newBuilder().build();
+ try (OutputStream tmpOutputStream = Files.newOutputStream(checksumFile.toPath())) {
+ emptyContainerInfo.writeTo(tmpOutputStream);
+ }
+
+ // Check state of original container.
+ checkContainerFilesPresent(data, 0);
+
+ //destination path
+ File exportTar = Files.createFile(
+ folder.toPath().resolve("export.tar")).toFile();
+ TarContainerPacker packer = new TarContainerPacker(NO_COMPRESSION);
+ //export the container
+ try (OutputStream fos = Files.newOutputStream(exportTar.toPath())) {
+ keyValueContainer.exportContainerData(fos, packer);
+ }
+
+ KeyValueContainerUtil.removeContainer(
+ keyValueContainer.getContainerData(), CONF);
+ keyValueContainer.delete();
+
+ // import container.
+ try (InputStream fis = Files.newInputStream(exportTar.toPath())) {
+ keyValueContainer.importContainerData(fis, packer);
+ }
+
+ // Make sure empty chunks dir was unpacked.
+ checkContainerFilesPresent(data, 0);
+ data = keyValueContainer.getContainerData();
+ ContainerProtos.ContainerChecksumInfo checksumInfo = ContainerChecksumTreeManager.readChecksumInfo(data);
+ assertFalse(checksumInfo.hasContainerMerkleTree());
+ // The import should not fail and the checksum should be 0
+ assertEquals(0, data.getDataChecksum());
+ // The checksum is not stored in rocksDB as the container merkle tree doesn't exist.
+ verifyAllDataChecksumsMatch(data, CONF);
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void testUnhealthyContainerImportExport(
ContainerTestVersionInfo versionInfo) throws Exception {
@@ -340,6 +390,18 @@ public void testContainerImportExport(ContainerTestVersionInfo versionInfo)
closeContainer();
populate(numberOfKeysToWrite);
+ // Create merkle tree and set data checksum to simulate actual key value container.
+ File checksumFile = ContainerChecksumTreeManager.getContainerChecksumFile(
+ keyValueContainer.getContainerData());
+ ContainerProtos.ContainerMerkleTree containerMerkleTreeWriterProto = buildTestTree(CONF).toProto();
+ keyValueContainerData.setDataChecksum(containerMerkleTreeWriterProto.getDataChecksum());
+ ContainerProtos.ContainerChecksumInfo containerInfo = ContainerProtos.ContainerChecksumInfo.newBuilder()
+ .setContainerID(containerId)
+ .setContainerMerkleTree(containerMerkleTreeWriterProto).build();
+ try (OutputStream tmpOutputStream = Files.newOutputStream(checksumFile.toPath())) {
+ containerInfo.writeTo(tmpOutputStream);
+ }
+
//destination path
File folderToExport = Files.createFile(
folder.toPath().resolve("export.tar")).toFile();
@@ -387,6 +449,9 @@ public void testContainerImportExport(ContainerTestVersionInfo versionInfo)
containerData.getMaxSize());
assertEquals(keyValueContainerData.getBytesUsed(),
containerData.getBytesUsed());
+ assertEquals(keyValueContainerData.getDataChecksum(), containerData.getDataChecksum());
+ verifyAllDataChecksumsMatch(containerData, CONF);
+
assertNotNull(containerData.getContainerFileChecksum());
assertNotEquals(containerData.ZERO_CHECKSUM, container.getContainerData().getContainerFileChecksum());
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
index 92e7a071cce..989dc0bddbc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
@@ -38,7 +38,6 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
@@ -200,9 +199,8 @@ public void testAllDataErrorsCollected(ContainerTestVersionInfo versionInfo) thr
checksumManager.writeContainerDataTree(containerData, result.getDataTree());
// This will read the corrupted tree from the disk, which represents the current state of the container, and
// compare it against the original healthy tree. The diff we get back should match the failures we injected.
- Optional<ContainerProtos.ContainerChecksumInfo> generatedChecksumInfo = checksumManager.read(containerData);
- assertTrue(generatedChecksumInfo.isPresent());
- ContainerDiffReport diffReport = checksumManager.diff(generatedChecksumInfo.get(), healthyChecksumInfo);
+ ContainerProtos.ContainerChecksumInfo generatedChecksumInfo = checksumManager.read(container.getContainerData());
+ ContainerDiffReport diffReport = checksumManager.diff(generatedChecksumInfo, healthyChecksumInfo);
LOG.info("Diff of healthy container with actual container {}", diffReport);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 503d8c0855d..17efe57e596 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -27,6 +27,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.GB;
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch;
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createBlockMetaData;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.assertj.core.api.Assertions.assertThat;
@@ -48,6 +49,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Sets;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -98,8 +100,7 @@
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
-import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
-import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.apache.hadoop.util.Time;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
@@ -383,7 +384,8 @@ public void testCloseInvalidContainer(ContainerLayoutVersion layoutVersion)
// Closing invalid container should return error response.
ContainerProtos.ContainerCommandResponseProto response =
keyValueHandler.handleCloseContainer(closeContainerRequest, container);
- assertTrue(ContainerChecksumTreeManager.checksumFileExist(container));
+ // Checksum will not be generated for an invalid container.
+ assertFalse(ContainerChecksumTreeManager.getContainerChecksumFile(kvData).exists());
assertEquals(ContainerProtos.Result.INVALID_CONTAINER_STATE,
response.getResult(),
@@ -679,17 +681,18 @@ public void testUpdateContainerChecksum(ContainerLayoutVersion layoutVersion) th
// Initially, container should have no checksum information.
assertEquals(0, containerData.getDataChecksum());
- assertFalse(checksumManager.read(containerData).isPresent());
+ assertFalse(checksumManager.read(containerData).hasContainerMerkleTree());
+ assertFalse(ContainerChecksumTreeManager.getContainerChecksumFile(containerData).exists());
assertEquals(0, icrCount.get());
// Update container with checksum information.
keyValueHandler.updateContainerChecksum(container, treeWriter);
// Check ICR sent. The ICR sender verifies that the expected checksum is present in the report.
assertEquals(1, icrCount.get());
- // Check checksum in memory.
- assertEquals(updatedDataChecksum, containerData.getDataChecksum());
+ // Check all data checksums are updated correctly.
+ verifyAllDataChecksumsMatch(containerData, conf);
// Check disk content.
- ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(containerData).get();
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(containerData);
assertTreesSortedAndMatch(treeWriter.toProto(), checksumInfo.getContainerMerkleTree());
}
@@ -818,7 +821,7 @@ private KeyValueHandler createKeyValueHandler(Path path) throws IOException {
// Register the on-demand container scanner with the container set used by the KeyValueHandler.
ContainerController controller = new ContainerController(containerSet,
Collections.singletonMap(ContainerType.KeyValueContainer, kvHandler));
- OnDemandContainerDataScanner onDemandScanner = new OnDemandContainerDataScanner(
+ OnDemandContainerScanner onDemandScanner = new OnDemandContainerScanner(
conf.getObject(ContainerScannerConfiguration.class), controller);
containerSet.registerOnDemandScanner(onDemandScanner);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
index 343c8dcfee8..8361959e6da 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
@@ -238,14 +238,14 @@ public void testMarkContainerUnhealthyInFailedVolume() throws IOException {
// be ignored.
hddsVolume.setState(StorageVolume.VolumeState.FAILED);
handler.markContainerUnhealthy(container, ContainerTestUtils.getUnhealthyDataScanResult());
- assertFalse(ContainerChecksumTreeManager.checksumFileExist(container));
+ assertFalse(ContainerChecksumTreeManager.getContainerChecksumFile(kvData).exists());
verify(mockIcrSender, never()).send(any());
// When volume is healthy, ICR should be sent when container is marked
// unhealthy.
hddsVolume.setState(StorageVolume.VolumeState.NORMAL);
handler.markContainerUnhealthy(container, ContainerTestUtils.getUnhealthyDataScanResult());
- assertTrue(ContainerChecksumTreeManager.checksumFileExist(container));
+ assertTrue(ContainerChecksumTreeManager.getContainerChecksumFile(kvData).exists());
verify(mockIcrSender, atMostOnce()).send(any());
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
index 7f5c4c6b27f..84528934750 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java
@@ -20,6 +20,8 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.Files.newInputStream;
import static java.nio.file.Files.newOutputStream;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
import static org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker.CONTAINER_FILE_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -44,16 +46,16 @@
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.Archiver;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeWriter;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.replication.CopyContainerCompression;
import org.apache.ozone.test.SpyInputStream;
import org.apache.ozone.test.SpyOutputStream;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -90,20 +92,21 @@ public class TestTarContainerPacker {
private TarContainerPacker packer;
- private static final Path SOURCE_CONTAINER_ROOT =
- Paths.get("target/test/data/packer-source-dir");
+ @TempDir
+ private Path sourceContainerRoot;
- private static final Path DEST_CONTAINER_ROOT =
- Paths.get("target/test/data/packer-dest-dir");
+ @TempDir
+ private Path destContainerRoot;
- private static final Path TEMP_DIR =
- Paths.get("target/test/data/packer-tmp-dir");
+ @TempDir
+ private Path tempDir;
private static final AtomicInteger CONTAINER_ID = new AtomicInteger(1);
private ContainerLayoutVersion layout;
private String schemaVersion;
private OzoneConfiguration conf;
+ private ContainerChecksumTreeManager checksumTreeManager;
private void initTests(ContainerTestVersionInfo versionInfo,
CopyContainerCompression compression) {
@@ -112,6 +115,7 @@ private void initTests(ContainerTestVersionInfo versionInfo,
this.conf = new OzoneConfiguration();
ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
packer = new TarContainerPacker(compression);
+ checksumTreeManager = new ContainerChecksumTreeManager(conf);
}
public static List<Arguments> getLayoutAndCompression() {
@@ -126,34 +130,16 @@ public static List<Arguments> getLayoutAndCompression() {
return parameterList;
}
- @BeforeAll
- public static void init() throws IOException {
- initDir(SOURCE_CONTAINER_ROOT);
- initDir(DEST_CONTAINER_ROOT);
- initDir(TEMP_DIR);
- }
-
- @AfterAll
- public static void cleanup() throws IOException {
- FileUtils.deleteDirectory(SOURCE_CONTAINER_ROOT.toFile());
- FileUtils.deleteDirectory(DEST_CONTAINER_ROOT.toFile());
- FileUtils.deleteDirectory(TEMP_DIR.toFile());
- }
-
- private static void initDir(Path path) throws IOException {
- if (path.toFile().exists()) {
- FileUtils.deleteDirectory(path.toFile());
- }
- Files.createDirectories(path);
- }
-
private KeyValueContainerData createContainer(Path dir) throws IOException {
- return createContainer(dir, true);
+ return createContainer(dir, true, true);
}
- private KeyValueContainerData createContainer(Path dir, boolean createDir)
+ private KeyValueContainerData createContainer(Path dir, boolean createDir, boolean incrementId)
throws IOException {
- long id = CONTAINER_ID.getAndIncrement();
+ long id = CONTAINER_ID.get();
+ if (incrementId) {
+ id = CONTAINER_ID.getAndIncrement();
+ }
Path containerDir = dir.resolve(String.valueOf(id));
Path dataDir = containerDir.resolve("chunks");
@@ -183,7 +169,7 @@ public void pack(ContainerTestVersionInfo versionInfo,
initTests(versionInfo, compression);
//GIVEN
KeyValueContainerData sourceContainerData =
- createContainer(SOURCE_CONTAINER_ROOT);
+ createContainer(sourceContainerRoot, true, false);
KeyValueContainer sourceContainer =
new KeyValueContainer(sourceContainerData, conf);
@@ -194,10 +180,15 @@ public void pack(ContainerTestVersionInfo versionInfo,
//sample chunk file in the chunk directory
writeChunkFile(sourceContainerData, TEST_CHUNK_FILE_NAME);
+ //write container checksum file in the metadata directory
+ ContainerMerkleTreeWriter treeWriter = buildTestTree(conf);
+ checksumTreeManager.writeContainerDataTree(sourceContainerData, treeWriter);
+ assertTrue(ContainerChecksumTreeManager.getContainerChecksumFile(sourceContainerData).exists());
+
//sample container descriptor file
writeDescriptor(sourceContainer);
- Path targetFile = TEMP_DIR.resolve("container.tar");
+ Path targetFile = tempDir.resolve("container.tar");
//WHEN: pack it
SpyOutputStream outputForPack =
@@ -239,7 +230,7 @@ public void pack(ContainerTestVersionInfo versionInfo,
inputForUnpackDescriptor.assertClosedExactlyOnce();
KeyValueContainerData destinationContainerData =
- createContainer(DEST_CONTAINER_ROOT, false);
+ createContainer(destContainerRoot, false, false);
KeyValueContainer destinationContainer =
new KeyValueContainer(destinationContainerData, conf);
@@ -249,7 +240,7 @@ public void pack(ContainerTestVersionInfo versionInfo,
new SpyInputStream(newInputStream(targetFile));
String descriptor = new String(
packer.unpackContainerData(destinationContainer, inputForUnpackData,
- TEMP_DIR, DEST_CONTAINER_ROOT.resolve(String.valueOf(
+ tempDir, destContainerRoot.resolve(String.valueOf(
destinationContainer.getContainerData().getContainerID()))),
UTF_8);
@@ -260,6 +251,11 @@ public void pack(ContainerTestVersionInfo versionInfo,
Paths.get(destinationContainerData.getChunksPath()),
TEST_CHUNK_FILE_NAME);
+ assertEquals(sourceContainerData.getContainerID(), destinationContainerData.getContainerID());
+ assertTrue(ContainerChecksumTreeManager.getContainerChecksumFile(destinationContainerData).exists());
+ assertTreesSortedAndMatch(checksumTreeManager.read(sourceContainerData).getContainerMerkleTree(),
+ checksumTreeManager.read(destinationContainerData).getContainerMerkleTree());
+
String containerFileData = new String(Files.readAllBytes(destinationContainer.getContainerFile().toPath()), UTF_8);
assertTrue(containerFileData.contains("RECOVERING"),
"The state of the container is not 'RECOVERING' in the container file");
@@ -276,7 +272,7 @@ public void unpackContainerDataWithValidRelativeDbFilePath(
initTests(versionInfo, compression);
//GIVEN
KeyValueContainerData sourceContainerData =
- createContainer(SOURCE_CONTAINER_ROOT);
+ createContainer(sourceContainerRoot);
String fileName = "sub/dir/" + TEST_DB_FILE_NAME;
File file = writeDbFile(sourceContainerData, fileName);
@@ -301,7 +297,7 @@ public void unpackContainerDataWithValidRelativeChunkFilePath(
initTests(versionInfo, compression);
//GIVEN
KeyValueContainerData sourceContainerData =
- createContainer(SOURCE_CONTAINER_ROOT);
+ createContainer(sourceContainerRoot);
String fileName = "sub/dir/" + TEST_CHUNK_FILE_NAME;
File file = writeChunkFile(sourceContainerData, fileName);
@@ -325,7 +321,7 @@ public void unpackContainerDataWithInvalidRelativeDbFilePath(
initTests(versionInfo, compression);
//GIVEN
KeyValueContainerData sourceContainerData =
- createContainer(SOURCE_CONTAINER_ROOT);
+ createContainer(sourceContainerRoot);
String fileName = "../db_file";
File file = writeDbFile(sourceContainerData, fileName);
@@ -346,7 +342,7 @@ public void unpackContainerDataWithInvalidRelativeChunkFilePath(
initTests(versionInfo, compression);
//GIVEN
KeyValueContainerData sourceContainerData =
- createContainer(SOURCE_CONTAINER_ROOT);
+ createContainer(sourceContainerRoot);
String fileName = "../chunk_file";
File file = writeChunkFile(sourceContainerData, fileName);
@@ -361,10 +357,10 @@ public void unpackContainerDataWithInvalidRelativeChunkFilePath(
private KeyValueContainerData unpackContainerData(File containerFile)
throws IOException {
try (InputStream input = newInputStream(containerFile.toPath())) {
- KeyValueContainerData data = createContainer(DEST_CONTAINER_ROOT, false);
+ KeyValueContainerData data = createContainer(destContainerRoot, false, true);
KeyValueContainer container = new KeyValueContainer(data, conf);
- packer.unpackContainerData(container, input, TEMP_DIR,
- DEST_CONTAINER_ROOT.resolve(String.valueOf(data.getContainerID())));
+ packer.unpackContainerData(container, input, tempDir,
+ destContainerRoot.resolve(String.valueOf(data.getContainerID())));
return data;
}
}
@@ -406,7 +402,7 @@ private File writeSingleFile(Path parentPath, String fileName,
private File packContainerWithSingleFile(File file, String entryName)
throws Exception {
- File targetFile = TEMP_DIR.resolve("container.tar").toFile();
+ File targetFile = tempDir.resolve("container.tar").toFile();
Path path = targetFile.toPath();
try (TarArchiveOutputStream archive = new TarArchiveOutputStream(packer.compress(newOutputStream(path)))) {
archive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
index 46d0c615737..6995b75f530 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
@@ -140,9 +140,9 @@ protected void checkChunkFileCount(int expected) {
*/
public static boolean isFileNotInUse(String filePath) {
try {
- Process process = new ProcessBuilder("fuser", filePath).start();
+ Process process = new ProcessBuilder("lsof", "-f", "--", filePath).start();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8))) {
- String output = reader.readLine(); // If fuser returns no output, the file is not in use
+ String output = reader.readLine(); // If lsof returns no output, the file is not in use
if (output == null) {
return true;
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
index 3a95af7f708..95475651d01 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
@@ -22,9 +22,11 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.File;
@@ -184,10 +186,11 @@ public void testWriteChunkForClosedContainer()
ChunkBuffer writeChunkData = ChunkBuffer.wrap(getData());
KeyValueContainer kvContainer = getKeyValueContainer();
KeyValueContainerData containerData = kvContainer.getContainerData();
- closedKeyValueContainer();
ContainerSet containerSet = newContainerSet();
containerSet.addContainer(kvContainer);
KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet);
+ keyValueHandler.markContainerForClose(kvContainer);
+ keyValueHandler.closeContainer(kvContainer);
keyValueHandler.writeChunkForClosedContainer(getChunkInfo(), getBlockID(), writeChunkData, kvContainer);
ChunkBufferToByteString readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer,
getBlockID(), getChunkInfo(), WRITE_STAGE);
@@ -228,12 +231,16 @@ public void testWriteChunkForClosedContainer()
@Test
public void testPutBlockForClosedContainer() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
KeyValueContainer kvContainer = getKeyValueContainer();
KeyValueContainerData containerData = kvContainer.getContainerData();
- closedKeyValueContainer();
ContainerSet containerSet = newContainerSet();
containerSet.addContainer(kvContainer);
KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet);
+ keyValueHandler.markContainerForClose(kvContainer);
+ keyValueHandler.closeContainer(kvContainer);
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, containerData.getState());
+ assertEquals(0L, containerData.getDataChecksum());
List<ContainerProtos.ChunkInfo> chunkInfoList = new ArrayList<>();
ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 0), 0L, 20L);
@@ -244,11 +251,13 @@ public void testPutBlockForClosedContainer() throws IOException {
ChunkBuffer chunkData = ContainerTestHelper.getData(20);
keyValueHandler.writeChunkForClosedContainer(info, getBlockID(), chunkData, kvContainer);
keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 1L, true);
+ keyValueHandler.updateAndGetContainerChecksumFromMetadata(kvContainer);
assertEquals(1L, containerData.getBlockCommitSequenceId());
assertEquals(1L, containerData.getBlockCount());
assertEquals(20L, containerData.getBytesUsed());
+ verifyAllDataChecksumsMatch(containerData, conf);
- try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
long localID = putBlockData.getLocalID();
BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
.get(containerData.getBlockKey(localID));
@@ -264,11 +273,15 @@ public void testPutBlockForClosedContainer() throws IOException {
chunkData = ContainerTestHelper.getData(20);
keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), chunkData, kvContainer);
keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true);
+ long previousDataChecksum = containerData.getDataChecksum();
+ keyValueHandler.updateAndGetContainerChecksumFromMetadata(kvContainer);
assertEquals(2L, containerData.getBlockCommitSequenceId());
assertEquals(1L, containerData.getBlockCount());
assertEquals(40L, containerData.getBytesUsed());
+ assertNotEquals(previousDataChecksum, containerData.getDataChecksum());
+ verifyAllDataChecksumsMatch(containerData, conf);
- try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
long localID = putBlockData.getLocalID();
BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
.get(containerData.getBlockKey(localID));
@@ -291,7 +304,7 @@ public void testPutBlockForClosedContainer() throws IOException {
// Old chunk size 20, new chunk size 30, difference 10. So bytesUsed should be 40 + 10 = 50
assertEquals(50L, containerData.getBytesUsed());
- try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
long localID = putBlockData.getLocalID();
BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
.get(containerData.getBlockKey(localID));
@@ -331,10 +344,6 @@ public KeyValueHandler createKeyValueHandler(ContainerSet containerSet)
return ContainerTestUtils.getKeyValueHandler(conf, dnUuid, containerSet, volumeSet);
}
- public void closedKeyValueContainer() {
- getKeyValueContainer().getContainerData().setState(ContainerProtos.ContainerDataProto.State.CLOSED);
- }
-
@Override
protected ContainerLayoutTestInfo getStrategy() {
return ContainerLayoutTestInfo.FILE_PER_BLOCK;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
index 93ef66680d8..508c472a7c8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
@@ -32,6 +32,7 @@
import static org.mockito.Mockito.atMostOnce;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -48,9 +49,12 @@
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
@@ -133,6 +137,17 @@ public void testUnhealthyContainersDetected() throws Exception {
verifyContainerMarkedUnhealthy(deletedContainer, never());
}
+ @Test
+ @Override
+ public void testUnhealthyContainersTriggersVolumeScan() throws Exception {
+ when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true);
+ try (MockedStatic<StorageVolumeUtil> mockedStatic = mockStatic(StorageVolumeUtil.class)) {
+ scanner.runIteration();
+ verifyContainerMarkedUnhealthy(corruptData, atLeastOnce());
+ mockedStatic.verify(() -> StorageVolumeUtil.onFailure(corruptData.getContainerData().getVolume()), times(1));
+ }
+ }
+
@Test
public void testScanTimestampUpdated() throws Exception {
scanner.runIteration();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
index abc3126f762..9b6c6aed3f0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
@@ -32,7 +32,9 @@
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.atMostOnce;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -45,9 +47,12 @@
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
+import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
@@ -130,6 +135,18 @@ public void testUnhealthyContainersDetected() throws Exception {
verifyContainerMarkedUnhealthy(openContainer, never());
}
+ @Test
+ @Override
+ public void testUnhealthyContainersTriggersVolumeScan() throws Exception {
+ when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true);
+ try (MockedStatic<StorageVolumeUtil> mockedStatic = mockStatic(StorageVolumeUtil.class)) {
+ scanner.runIteration();
+ verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce());
+ mockedStatic.verify(() ->
+ StorageVolumeUtil.onFailure(openCorruptMetadata.getContainerData().getVolume()), times(1));
+ }
+ }
+
@Test
@Override
public void testUnhealthyContainerRescanned() throws Exception {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
index ec5c6743e72..69415972d00 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
@@ -20,12 +20,15 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.DELETED;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.verifyAllDataChecksumsMatch;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getKeyValueHandler;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.anyList;
@@ -39,8 +42,10 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.UUID;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -49,6 +54,9 @@
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeWriter;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
@@ -66,6 +74,7 @@
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
import org.apache.hadoop.util.Time;
@@ -92,6 +101,7 @@ public class TestContainerReader {
private ContainerLayoutVersion layout;
private String schemaVersion;
+ private KeyValueHandler keyValueHandler;
@TempDir
private Path tempDir;
@@ -140,6 +150,7 @@ private void setup(ContainerTestVersionInfo versionInfo) throws Exception {
// so it does not affect the ContainerReader, which avoids using the cache
// at startup for performance reasons.
ContainerCache.getInstance(conf).shutdownCache();
+ keyValueHandler = getKeyValueHandler(conf, UUID.randomUUID().toString(), containerSet, volumeSet);
}
@AfterEach
@@ -609,6 +620,159 @@ public void testMarkedDeletedContainerCleared(
}
}
+ @ContainerTestVersionInfo.ContainerTest
+ public void testContainerLoadingWithMerkleTreePresent(ContainerTestVersionInfo versionInfo)
+ throws Exception {
+ setLayoutAndSchemaVersion(versionInfo);
+ setup(versionInfo);
+
+ // Create a container with blocks and write MerkleTree
+ KeyValueContainer container = createContainer(10L);
+ KeyValueContainerData containerData = container.getContainerData();
+ ContainerMerkleTreeWriter treeWriter = ContainerMerkleTreeTestUtils.buildTestTree(conf);
+ ContainerChecksumTreeManager checksumManager = keyValueHandler.getChecksumManager();
+ List<Long> deletedBlockIds = Arrays.asList(1L, 2L, 3L);
+ checksumManager.markBlocksAsDeleted(containerData, deletedBlockIds);
+ keyValueHandler.updateContainerChecksum(container, treeWriter);
+ long expectedDataChecksum = checksumManager.read(containerData).getContainerMerkleTree().getDataChecksum();
+
+ // Test container loading
+ ContainerCache.getInstance(conf).shutdownCache();
+ ContainerReader containerReader = new ContainerReader(volumeSet, hddsVolume, containerSet, conf, true);
+ containerReader.run();
+
+ // Verify container was loaded successfully and data checksum is set
+ Container<?> loadedContainer = containerSet.getContainer(10L);
+ assertNotNull(loadedContainer);
+ KeyValueContainerData loadedData = (KeyValueContainerData) loadedContainer.getContainerData();
+ assertNotSame(containerData, loadedData);
+ assertEquals(expectedDataChecksum, loadedData.getDataChecksum());
+ ContainerProtos.ContainerChecksumInfo loadedChecksumInfo =
+ ContainerChecksumTreeManager.readChecksumInfo(loadedData);
+ verifyAllDataChecksumsMatch(loadedData, conf);
+
+ // Verify the deleted block IDs match what we set
+ List<Long> loadedDeletedBlockIds = loadedChecksumInfo.getDeletedBlocksList().stream()
+ .map(ContainerProtos.BlockMerkleTree::getBlockID)
+ .sorted()
+ .collect(Collectors.toList());
+ assertEquals(3, loadedChecksumInfo.getDeletedBlocksCount());
+ assertEquals(deletedBlockIds, loadedDeletedBlockIds);
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testContainerLoadingWithMerkleTreeFallbackToRocksDB(ContainerTestVersionInfo versionInfo)
+ throws Exception {
+ setLayoutAndSchemaVersion(versionInfo);
+ setup(versionInfo);
+
+ KeyValueContainer container = createContainer(11L);
+ KeyValueContainerData containerData = container.getContainerData();
+ ContainerMerkleTreeWriter treeWriter = ContainerMerkleTreeTestUtils.buildTestTree(conf);
+ ContainerChecksumTreeManager checksumManager = new ContainerChecksumTreeManager(conf);
+ ContainerProtos.ContainerChecksumInfo checksumInfo =
+ checksumManager.writeContainerDataTree(containerData, treeWriter);
+ long dataChecksum = checksumInfo.getContainerMerkleTree().getDataChecksum();
+
+ // Verify no checksum in RocksDB initially
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+ Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+ assertNull(dbDataChecksum);
+ }
+ ContainerCache.getInstance(conf).shutdownCache();
+
+ // Test container loading - should read from MerkleTree and store in RocksDB
+ ContainerReader containerReader = new ContainerReader(volumeSet, hddsVolume, containerSet, conf, true);
+ containerReader.run();
+
+ // Verify container uses checksum from MerkleTree
+ Container<?> loadedContainer = containerSet.getContainer(11L);
+ assertNotNull(loadedContainer);
+ KeyValueContainerData loadedData = (KeyValueContainerData) loadedContainer.getContainerData();
+ assertNotSame(containerData, loadedData);
+ assertEquals(dataChecksum, loadedData.getDataChecksum());
+
+ // Verify checksum was stored in RocksDB as fallback
+ verifyAllDataChecksumsMatch(loadedData, conf);
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testContainerLoadingWithNoChecksumAnywhere(ContainerTestVersionInfo versionInfo)
+ throws Exception {
+ setLayoutAndSchemaVersion(versionInfo);
+ setup(versionInfo);
+
+ KeyValueContainer container = createContainer(12L);
+ KeyValueContainerData containerData = container.getContainerData();
+ // Verify no checksum in RocksDB
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+ Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+ assertNull(dbDataChecksum);
+ }
+
+ File checksumFile = ContainerChecksumTreeManager.getContainerChecksumFile(containerData);
+ assertFalse(checksumFile.exists());
+
+ // Test container loading - should default to 0
+ ContainerCache.getInstance(conf).shutdownCache();
+ ContainerReader containerReader = new ContainerReader(volumeSet, hddsVolume, containerSet, conf, true);
+ containerReader.run();
+
+ // Verify container loads with default checksum of 0
+ Container<?> loadedContainer = containerSet.getContainer(12L);
+ assertNotNull(loadedContainer);
+ KeyValueContainerData loadedData = (KeyValueContainerData) loadedContainer.getContainerData();
+ assertNotSame(containerData, loadedData);
+ assertEquals(0L, loadedData.getDataChecksum());
+
+ // The checksum is not stored in rocksDB as the checksum file doesn't exist.
+ verifyAllDataChecksumsMatch(loadedData, conf);
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testContainerLoadingWithoutMerkleTree(ContainerTestVersionInfo versionInfo)
+ throws Exception {
+ setLayoutAndSchemaVersion(versionInfo);
+ setup(versionInfo);
+
+ KeyValueContainer container = createContainer(13L);
+ KeyValueContainerData containerData = container.getContainerData();
+ ContainerMerkleTreeWriter treeWriter = new ContainerMerkleTreeWriter();
+ keyValueHandler.updateContainerChecksum(container, treeWriter);
+ // Create an empty checksum file that exists but has no valid merkle tree
+ assertTrue(ContainerChecksumTreeManager.getContainerChecksumFile(containerData).exists());
+
+ // Verify no checksum in RocksDB initially
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+ Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+ assertNull(dbDataChecksum);
+ }
+
+ ContainerCache.getInstance(conf).shutdownCache();
+
+ // Test container loading - should handle when checksum file is present without the container merkle tree and
+ // default to 0.
+ ContainerReader containerReader = new ContainerReader(volumeSet, hddsVolume, containerSet, conf, true);
+ containerReader.run();
+
+ // Verify container loads with default checksum of 0 when checksum file doesn't have merkle tree
+ Container<?> loadedContainer = containerSet.getContainer(13L);
+ assertNotNull(loadedContainer);
+ KeyValueContainerData loadedData = (KeyValueContainerData) loadedContainer.getContainerData();
+ assertNotSame(containerData, loadedData);
+ assertEquals(0L, loadedData.getDataChecksum());
+ verifyAllDataChecksumsMatch(loadedData, conf);
+ }
+
+ private KeyValueContainer createContainer(long containerId) throws Exception {
+ KeyValueContainerData containerData = new KeyValueContainerData(containerId, layout,
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(), datanodeId.toString());
+ containerData.setState(ContainerProtos.ContainerDataProto.State.CLOSED);
+ KeyValueContainer container = new KeyValueContainer(containerData, conf);
+ container.create(volumeSet, volumeChoosingPolicy, clusterId);
+ return container;
+ }
+
private long addDbEntry(KeyValueContainerData containerData)
throws Exception {
try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java
index f3537480970..7bd45c3b503 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java
@@ -31,6 +31,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
@@ -130,6 +131,9 @@ public abstract void testPreviouslyScannedContainerIsScanned()
@Test
public abstract void testChecksumUpdateFailure() throws Exception;
+ @Test
+ public abstract void testUnhealthyContainersTriggersVolumeScan() throws Exception;
+
// HELPER METHODS
protected void setScannedTimestampOld(Container<ContainerData> container) {
@@ -198,6 +202,10 @@ private ContainerController mockContainerController() {
MetadataScanResult healthyMetadata = getHealthyMetadataScanResult();
MetadataScanResult unhealthyMetadata = getUnhealthyMetadataScanResult();
+ File volLocation = mock(File.class);
+ when(volLocation.getPath()).thenReturn("/temp/volume-testcontainerscanner");
+ when(vol.getStorageDir()).thenReturn(volLocation);
+
// healthy container
ContainerTestUtils.setupMockContainer(healthy,
true, healthyMetadata, healthyData,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
similarity index 84%
rename from hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java
rename to hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
index 42b3da8050d..69b117db123 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
@@ -35,7 +35,6 @@
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -53,6 +52,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ScanResult;
+import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -64,16 +64,17 @@
* Unit tests for the on-demand container scanner.
*/
@MockitoSettings(strictness = Strictness.LENIENT)
-public class TestOnDemandContainerDataScanner extends
+public class TestOnDemandContainerScanner extends
TestContainerScannersAbstract {
-
- private OnDemandContainerDataScanner onDemandScanner;
+
+ private OnDemandContainerScanner onDemandScanner;
+ private static final String TEST_SCAN = "Test Scan";
@Override
@BeforeEach
public void setup() {
super.setup();
- onDemandScanner = new OnDemandContainerDataScanner(conf, controller);
+ onDemandScanner = new OnDemandContainerScanner(conf, controller);
}
@Test
@@ -88,7 +89,7 @@ public void testRecentlyScannedContainerIsSkipped() throws Exception {
public void testBypassScanGap() throws Exception {
setScannedTimestampRecent(healthy);
- Optional<Future<?>> scanFutureOptional = onDemandScanner.scanContainerWithoutGap(healthy);
+ Optional<Future<?>> scanFutureOptional = onDemandScanner.scanContainerWithoutGap(healthy, TEST_SCAN);
assertTrue(scanFutureOptional.isPresent());
Future<?> scanFuture = scanFutureOptional.get();
scanFuture.get();
@@ -122,7 +123,7 @@ public void tearDown() {
@Test
public void testScanTimestampUpdated() throws Exception {
- Optional<Future<?>> scanFuture = onDemandScanner.scanContainer(healthy);
+ Optional<Future<?>> scanFuture = onDemandScanner.scanContainer(healthy, TEST_SCAN);
assertTrue(scanFuture.isPresent());
scanFuture.get().get();
verify(controller, atLeastOnce())
@@ -130,7 +131,7 @@ public void testScanTimestampUpdated() throws Exception {
eq(healthy.getContainerData().getContainerID()), any());
// Metrics for deleted container should not be updated.
- scanFuture = onDemandScanner.scanContainer(healthy);
+ scanFuture = onDemandScanner.scanContainer(healthy, TEST_SCAN);
assertTrue(scanFuture.isPresent());
scanFuture.get().get();
verify(controller, never())
@@ -156,11 +157,11 @@ public void testSameContainerQueuedMultipleTimes() throws Exception {
latch.await();
return getUnhealthyDataScanResult();
});
- Optional<Future<?>> onGoingScan = onDemandScanner.scanContainer(corruptData);
+ Optional<Future<?>> onGoingScan = onDemandScanner.scanContainer(corruptData, TEST_SCAN);
assertTrue(onGoingScan.isPresent());
assertFalse(onGoingScan.get().isDone());
//When scheduling the same container again
- Optional<Future<?>> secondScan = onDemandScanner.scanContainer(corruptData);
+ Optional<Future<?>> secondScan = onDemandScanner.scanContainer(corruptData, TEST_SCAN);
//Then the second scan is not scheduled and the first scan can still finish
assertFalse(secondScan.isPresent());
latch.countDown();
@@ -169,22 +170,44 @@ public void testSameContainerQueuedMultipleTimes() throws Exception {
eq(corruptData.getContainerData().getContainerID()), any());
}
+ @Test
+ public void testSameOpenContainerQueuedMultipleTimes() throws Exception {
+ //Given a container that has not finished scanning
+ CountDownLatch latch = new CountDownLatch(1);
+ when(openCorruptMetadata.scanMetaData())
+ .thenAnswer((Answer<ScanResult>) invocation -> {
+ latch.await();
+ return getUnhealthyDataScanResult();
+ });
+ Optional<Future<?>> onGoingScan = onDemandScanner.scanContainer(openCorruptMetadata, TEST_SCAN);
+ assertTrue(onGoingScan.isPresent());
+ assertFalse(onGoingScan.get().isDone());
+ //When scheduling the same container again
+ Optional<Future<?>> secondScan = onDemandScanner.scanContainer(openCorruptMetadata, TEST_SCAN);
+ //Then the second scan is not scheduled and the first scan can still finish
+ assertFalse(secondScan.isPresent());
+ latch.countDown();
+ onGoingScan.get().get();
+ verify(controller, atLeastOnce()).markContainerUnhealthy(
+ eq(openCorruptMetadata.getContainerData().getContainerID()), any());
+ }
+
@Test
@Override
public void testScannerMetrics() throws Exception {
ArrayList<Optional<Future<?>>> resultFutureList = Lists.newArrayList();
- resultFutureList.add(onDemandScanner.scanContainer(corruptData));
- resultFutureList.add(onDemandScanner.scanContainer(openContainer));
- resultFutureList.add(onDemandScanner.scanContainer(openCorruptMetadata));
- resultFutureList.add(onDemandScanner.scanContainer(healthy));
+ resultFutureList.add(onDemandScanner.scanContainer(corruptData, TEST_SCAN));
+ resultFutureList.add(onDemandScanner.scanContainer(openContainer, TEST_SCAN));
+ resultFutureList.add(onDemandScanner.scanContainer(openCorruptMetadata, TEST_SCAN));
+ resultFutureList.add(onDemandScanner.scanContainer(healthy, TEST_SCAN));
// Deleted containers will not count towards the scan count metric.
- resultFutureList.add(onDemandScanner.scanContainer(deletedContainer));
+ resultFutureList.add(onDemandScanner.scanContainer(deletedContainer, TEST_SCAN));
waitOnScannerToFinish(resultFutureList);
OnDemandScannerMetrics metrics = onDemandScanner.getMetrics();
//Containers with shouldScanData = false shouldn't increase
// the number of scanned containers
assertEquals(0, metrics.getNumUnHealthyContainers());
- assertEquals(2, metrics.getNumContainersScanned());
+ assertEquals(4, metrics.getNumContainersScanned());
}
@Test
@@ -199,17 +222,12 @@ public void testScannerMetricsUnregisters() {
@Test
@Override
public void testUnhealthyContainersDetected() throws Exception {
- // Without initialization,
- // there shouldn't be interaction with containerController
- onDemandScanner.scanContainer(corruptData);
- verifyNoInteractions(controller);
-
scanContainer(healthy);
verifyContainerMarkedUnhealthy(healthy, never());
scanContainer(corruptData);
verifyContainerMarkedUnhealthy(corruptData, atLeastOnce());
scanContainer(openCorruptMetadata);
- verifyContainerMarkedUnhealthy(openCorruptMetadata, never());
+ verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce());
scanContainer(openContainer);
verifyContainerMarkedUnhealthy(openContainer, never());
// Deleted containers should not be marked unhealthy
@@ -252,7 +270,7 @@ public void testShutdownDuringScan() throws Exception {
});
// Start the blocking scan.
- onDemandScanner.scanContainer(healthy);
+ onDemandScanner.scanContainer(healthy, TEST_SCAN);
// Shut down the on demand scanner. This will interrupt the blocked scan
// on the healthy container.
onDemandScanner.shutdown();
@@ -325,8 +343,18 @@ public void testMerkleTreeWritten() throws Exception {
}
}
+ @Test
+ @Override
+ public void testUnhealthyContainersTriggersVolumeScan() throws Exception {
+ when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true);
+ LogCapturer logCapturer = LogCapturer.captureLogs(OnDemandContainerScanner.class);
+ scanContainer(corruptData);
+ verifyContainerMarkedUnhealthy(corruptData, times(1));
+ assertTrue(logCapturer.getOutput().contains("Triggering scan of volume"));
+ }
+
private void scanContainer(Container<?> container) throws Exception {
- Optional<Future<?>> scanFuture = onDemandScanner.scanContainer(container);
+ Optional<Future<?>> scanFuture = onDemandScanner.scanContainer(container, TEST_SCAN);
if (scanFuture.isPresent()) {
scanFuture.get().get();
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
index e6fe1e05dab..52f184d3e38 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
@@ -23,9 +23,13 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
@@ -35,6 +39,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.HashSet;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Semaphore;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
@@ -51,6 +56,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
@@ -58,6 +64,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.util.Time;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -74,29 +81,41 @@ class TestContainerImporter {
private OzoneConfiguration conf;
private VolumeChoosingPolicy volumeChoosingPolicy;
+ private KeyValueContainerData containerData;
+ private KeyValueContainer container;
+ private ContainerController controllerMock;
+ private long containerId = 1;
+ private ContainerSet containerSet;
+ private MutableVolumeSet volumeSet;
+ private ContainerImporter containerImporter;
@BeforeEach
- void setup() {
+ void setup() throws IOException {
conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, tempDir.getAbsolutePath());
volumeChoosingPolicy = VolumeChoosingPolicyFactory.getPolicy(conf);
+ // create container
+ containerData = new KeyValueContainerData(containerId,
+ ContainerLayoutVersion.FILE_PER_BLOCK, 100, "test", "test");
+ container = new KeyValueContainer(containerData, conf);
+ controllerMock = mock(ContainerController.class);
+ when(controllerMock.importContainer(any(ContainerData.class), any(), any())).thenAnswer(
+ i -> {
+ containerData = i.getArgument(0);
+ container = new KeyValueContainer(containerData, conf);
+ return container;
+ });
+ containerSet = spy(newContainerSet(0));
+ volumeSet = new MutableVolumeSet("test", conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ // create containerImporter object
+ containerImporter = new ContainerImporter(conf,
+ containerSet, controllerMock, volumeSet, volumeChoosingPolicy);
}
@Test
void importSameContainerWhenAlreadyImport() throws Exception {
- long containerId = 1;
- // create container
- KeyValueContainerData containerData = new KeyValueContainerData(containerId,
- ContainerLayoutVersion.FILE_PER_BLOCK, 100, "test", "test");
- KeyValueContainer container = new KeyValueContainer(containerData, conf);
- ContainerController controllerMock = mock(ContainerController.class);
- // create containerImporter object
- ContainerSet containerSet = newContainerSet(0);
containerSet.addContainer(container);
- MutableVolumeSet volumeSet = new MutableVolumeSet("test", conf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
- ContainerImporter containerImporter = new ContainerImporter(conf,
- containerSet, controllerMock, volumeSet, volumeChoosingPolicy);
File tarFile = new File("dummy.tar");
// second import should fail immediately
StorageContainerException ex = assertThrows(StorageContainerException.class,
@@ -108,25 +127,13 @@ void importSameContainerWhenAlreadyImport() throws Exception {
@Test
void importSameContainerWhenFirstInProgress() throws Exception {
- long containerId = 1;
- // create container
- KeyValueContainerData containerData = new KeyValueContainerData(containerId,
- ContainerLayoutVersion.FILE_PER_BLOCK, 100, "test", "test");
- KeyValueContainer container = new KeyValueContainer(containerData, conf);
// mock controller for return container data with delay
- ContainerController controllerMock = mock(ContainerController.class);
Semaphore semaphore = new Semaphore(0);
when(controllerMock.importContainer(any(), any(), any()))
.thenAnswer((invocation) -> {
semaphore.acquire();
return container;
});
- // create containerImporter object
- ContainerSet containerSet = newContainerSet(0);
- MutableVolumeSet volumeSet = new MutableVolumeSet("test", conf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
- ContainerImporter containerImporter = new ContainerImporter(conf,
- containerSet, controllerMock, volumeSet, volumeChoosingPolicy);
// run import async first time having delay
File tarFile = containerTarFile(containerId, containerData);
CompletableFuture.runAsync(() -> {
@@ -152,45 +159,69 @@ void importSameContainerWhenFirstInProgress() throws Exception {
@Test
public void testInconsistentChecksumContainerShouldThrowError() throws Exception {
- // create container
- long containerId = 1;
- KeyValueContainerData containerData = spy(new KeyValueContainerData(containerId,
+ // create container with mock to return different checksums
+ KeyValueContainerData data = spy(new KeyValueContainerData(containerId,
ContainerLayoutVersion.FILE_PER_BLOCK, 100, "test", "test"));
// mock to return different checksum
- when(containerData.getContainerFileChecksum()).thenReturn("checksum1", "checksum2");
- // create containerImporter object
- ContainerController controllerMock = mock(ContainerController.class);
- ContainerSet containerSet = newContainerSet(0);
- MutableVolumeSet volumeSet = new MutableVolumeSet("test", conf, null,
- StorageVolume.VolumeType.DATA_VOLUME, null);
- ContainerImporter containerImporter = spy(new ContainerImporter(conf,
+ when(data.getContainerFileChecksum()).thenReturn("checksum1", "checksum2");
+ // create containerImporter object with mock
+ ContainerImporter importer = spy(new ContainerImporter(conf,
containerSet, controllerMock, volumeSet, volumeChoosingPolicy));
TarContainerPacker packer = mock(TarContainerPacker.class);
when(packer.unpackContainerDescriptor(any())).thenReturn("test".getBytes(
StandardCharsets.UTF_8));
- when(containerImporter.getPacker(any())).thenReturn(packer);
+ when(importer.getPacker(any())).thenReturn(packer);
- doReturn(containerData).when(containerImporter).getKeyValueContainerData(any(byte[].class));
- when(containerImporter.getImportContainerProgress()).thenReturn(new HashSet<>());
+ doReturn(data).when(importer).getKeyValueContainerData(any(byte[].class));
+ when(importer.getImportContainerProgress()).thenReturn(new HashSet<>());
File tarFile = File.createTempFile("temp_" + System
.currentTimeMillis(), ".tar");
StorageContainerException scException =
assertThrows(StorageContainerException.class,
- () -> containerImporter.importContainer(containerId,
+ () -> importer.importContainer(containerId,
tarFile.toPath(), null, NO_COMPRESSION));
Assertions.assertTrue(scException.getMessage().
contains("Container checksum error"));
}
- private File containerTarFile(
- long containerId, ContainerData containerData) throws IOException {
+ @Test
+ public void testImportContainerTriggersOnDemandScanner() throws Exception {
+ // create containerImporter object
+ HddsVolume targetVolume = mock(HddsVolume.class);
+ doNothing().when(targetVolume).incrementUsedSpace(anyLong());
+
+ // import the container
+ File tarFile = containerTarFile(containerId, containerData);
+ containerImporter.importContainer(containerId, tarFile.toPath(),
+ targetVolume, NO_COMPRESSION);
+
+ verify(containerSet, atLeastOnce()).scanContainer(containerId, "Imported container");
+ }
+
+ @Test
+ public void testImportContainerResetsLastScanTime() throws Exception {
+ containerData.setDataScanTimestamp(Time.monotonicNow());
+
+ // create containerImporter object
+ HddsVolume targetVolume = mock(HddsVolume.class);
+ doNothing().when(targetVolume).incrementUsedSpace(anyLong());
+
+ // import the container
+ File tarFile = containerTarFile(containerId, containerData);
+ containerImporter.importContainer(containerId, tarFile.toPath(),
+ targetVolume, NO_COMPRESSION);
+
+ assertEquals(Optional.empty(), containerData.lastDataScanTime());
+ }
+
+ private File containerTarFile(long id, ContainerData data) throws IOException {
File yamlFile = new File(tempDir, "container.yaml");
- ContainerDataYaml.createContainerFile(containerData, yamlFile);
+ ContainerDataYaml.createContainerFile(data, yamlFile);
File tarFile = new File(tempDir,
- ContainerUtils.getContainerTarName(containerId));
+ ContainerUtils.getContainerTarName(id));
try (OutputStream output = Files.newOutputStream(tarFile.toPath())) {
ArchiveOutputStream<TarArchiveEntry> archive = new TarArchiveOutputStream(output);
TarArchiveEntry entry = archive.createArchiveEntry(yamlFile,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index f38b4acb8e1..f439e6a1d34 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -787,6 +787,44 @@ public void testPriorityOrdering(ContainerLayoutVersion layout)
supervisor.getInFlightReplications(BlockingTask.class));
}
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testReconcileContainerCommandDeduplication() {
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .build();
+
+ try {
+ final long containerID = 10L;
+
+ // Create reconcile commands with the same container ID but different peers
+ ReconcileContainerCommand command1 = new ReconcileContainerCommand(containerID, Collections.singleton(
+ MockDatanodeDetails.randomDatanodeDetails()));
+ ReconcileContainerCommand command2 = new ReconcileContainerCommand(containerID, Collections.singleton(
+ MockDatanodeDetails.randomDatanodeDetails()));
+ assertEquals(command1, command2);
+ ReconcileContainerTask task1 = new ReconcileContainerTask(
+ mock(ContainerController.class),
+ mock(DNContainerOperationClient.class),
+ command1);
+ ReconcileContainerTask task2 = new ReconcileContainerTask(
+ mock(ContainerController.class),
+ mock(DNContainerOperationClient.class),
+ command2);
+
+ // Add first task - should be accepted
+ supervisor.addTask(task1);
+ assertEquals(1, supervisor.getTotalInFlightReplications());
+ assertEquals(1, supervisor.getReplicationQueuedCount());
+
+ // Add second task with same container ID but different peers - should be deduplicated
+ supervisor.addTask(task2);
+ assertEquals(1, supervisor.getTotalInFlightReplications());
+ assertEquals(1, supervisor.getReplicationQueuedCount());
+ } finally {
+ supervisor.stop();
+ }
+ }
+
private static class BlockingTask extends AbstractReplicationTask {
private final CountDownLatch runningLatch;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToContainerIdsTable.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToContainerIdsTable.java
new file mode 100644
index 00000000000..e6ddf977340
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToContainerIdsTable.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.upgrade;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.OPEN;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.InetSocketAddress;
+import java.nio.file.Path;
+import java.util.Collections;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TypedTable;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.container.common.ScmTestMock;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.metadata.ContainerCreateInfo;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerDBDefinition;
+import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Tests upgrading a single datanode from HBASE_SUPPORT to CONTAINERID_TABLE_SCHEMA_CHANGE.
+ */
+public class TestDatanodeUpgradeToContainerIdsTable {
+ @TempDir
+ private Path tempFolder;
+
+ private DatanodeStateMachine dsm;
+ private ContainerDispatcher dispatcher;
+ private OzoneConfiguration conf;
+ private static final String CLUSTER_ID = "clusterID";
+
+ private RPC.Server scmRpcServer;
+ private InetSocketAddress address;
+
+ private void initTests() throws Exception {
+ conf = new OzoneConfiguration();
+ setup();
+ }
+
+ private void setup() throws Exception {
+ address = SCMTestUtils.getReuseableAddress();
+ conf.setSocketAddr(ScmConfigKeys.OZONE_SCM_NAMES, address);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ tempFolder.toString());
+ }
+
+ @AfterEach
+ public void teardown() throws Exception {
+ if (scmRpcServer != null) {
+ scmRpcServer.stop();
+ }
+
+ if (dsm != null) {
+ dsm.close();
+ }
+ }
+
+ @Test
+ public void testContainerTableAccessBeforeAndAfterUpgrade() throws Exception {
+ initTests();
+ // start DN and SCM
+ scmRpcServer = SCMTestUtils.startScmRpcServer(conf, new ScmTestMock(CLUSTER_ID), address, 10);
+ UpgradeTestHelper.addHddsVolume(conf, tempFolder);
+ dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
+ HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion());
+ dispatcher = dsm.getContainer().getDispatcher();
+ final Pipeline pipeline = MockPipeline.createPipeline(Collections.singletonList(dsm.getDatanodeDetails()));
+
+ // add a container
+ final long containerID = UpgradeTestHelper.addContainer(dispatcher, pipeline);
+ Container<?> container = dsm.getContainer().getContainerSet().getContainer(containerID);
+ assertEquals(OPEN, container.getContainerData().getState());
+
+ // check if the containerIds table is in old format
+ WitnessedContainerMetadataStore metadataStore = dsm.getContainer().getWitnessedContainerMetadataStore();
+ TypedTable<ContainerID, String> tableWithStringCodec = metadataStore.getStore().getTable(
+ metadataStore.getContainerCreateInfoTable().getName(), ContainerID.getCodec(), StringCodec.get());
+ assertEquals("containerIds", metadataStore.getContainerCreateInfoTable().getName());
+ assertEquals(OPEN.name(), tableWithStringCodec.get(ContainerID.valueOf(containerID)));
+
+ // close container to allow upgrade.
+ UpgradeTestHelper.closeContainer(dispatcher, containerID, pipeline);
+
+ dsm.finalizeUpgrade();
+ assertTrue(dsm.getLayoutVersionManager().isAllowed(HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE));
+ assertEquals(WitnessedContainerDBDefinition.CONTAINER_CREATE_INFO_TABLE_DEF.getName(),
+ metadataStore.getContainerCreateInfoTable().getName());
+ ContainerCreateInfo containerCreateInfo = metadataStore.getContainerCreateInfoTable().get(
+ ContainerID.valueOf(containerID));
+ // state is always open as state is update while create container only.
+ assertEquals(OPEN, containerCreateInfo.getState());
+ }
+
+ @Test
+ public void testContainerTableFinalizeRetry() throws Exception {
+ initTests();
+ // start DN and SCM
+ scmRpcServer = SCMTestUtils.startScmRpcServer(conf, new ScmTestMock(CLUSTER_ID), address, 10);
+ UpgradeTestHelper.addHddsVolume(conf, tempFolder);
+ dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
+ HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion());
+ dispatcher = dsm.getContainer().getDispatcher();
+ final Pipeline pipeline = MockPipeline.createPipeline(Collections.singletonList(dsm.getDatanodeDetails()));
+
+ // add a container
+ final long containerID = UpgradeTestHelper.addContainer(dispatcher, pipeline);
+ Container<?> container = dsm.getContainer().getContainerSet().getContainer(containerID);
+ assertEquals(OPEN, container.getContainerData().getState());
+
+ // check if the containerIds table is in old format
+ WitnessedContainerMetadataStore metadataStore = dsm.getContainer().getWitnessedContainerMetadataStore();
+ TypedTable<ContainerID, String> tableWithStringCodec = metadataStore.getStore().getTable(
+ metadataStore.getContainerCreateInfoTable().getName(), ContainerID.getCodec(), StringCodec.get());
+ assertEquals("containerIds", metadataStore.getContainerCreateInfoTable().getName());
+ assertEquals(OPEN.name(), tableWithStringCodec.get(ContainerID.valueOf(containerID)));
+
+ // add few more container entries to containerIds table as dummy
+ for (int i = 0; i < 10; i++) {
+ long containerIDWithDummy = containerID + i + 1;
+ tableWithStringCodec.put(ContainerID.valueOf(containerIDWithDummy), OPEN.name());
+ }
+
+ // close container to allow upgrade.
+ UpgradeTestHelper.closeContainer(dispatcher, containerID, pipeline);
+
+ // trigger one upgrade which is not persisted to metastore, partial upgrade
+ ContainerTableSchemaFinalizeAction upgradeAction = new ContainerTableSchemaFinalizeAction();
+ upgradeAction.execute(dsm);
+
+ Table<ContainerID, ContainerCreateInfo> currTable =
+ WitnessedContainerDBDefinition.CONTAINER_CREATE_INFO_TABLE_DEF.getTable(metadataStore.getStore());
+ assertEquals(11, getTableEntryCount(currTable)); // 1 original + 10 dummy entries
+
+ // cleanup entry and again upgrade
+ for (int i = 0; i < 10; i++) {
+ long containerIDWithDummy = containerID + i + 1;
+ tableWithStringCodec.delete(ContainerID.valueOf(containerIDWithDummy));
+ }
+
+ // trigger another upgrade which will update metainfo for upgrade
+ dsm.finalizeUpgrade();
+ assertTrue(dsm.getLayoutVersionManager().isAllowed(HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE));
+ assertEquals(WitnessedContainerDBDefinition.CONTAINER_CREATE_INFO_TABLE_DEF.getName(),
+ metadataStore.getContainerCreateInfoTable().getName());
+ ContainerCreateInfo containerCreateInfo
+ = metadataStore.getContainerCreateInfoTable().get(ContainerID.valueOf(containerID));
+ // state is always open as state is update while create container only.
+ assertEquals(OPEN, containerCreateInfo.getState());
+
+ assertEquals(1, getTableEntryCount(currTable)); // 1 original + 10 dummy entries
+ }
+
+ private static int getTableEntryCount(Table<ContainerID, ContainerCreateInfo> currTable) throws Exception {
+ int count = 0;
+ try (Table.KeyValueIterator<ContainerID, ContainerCreateInfo> curTblItr = currTable.iterator()) {
+ while (curTblItr.hasNext()) {
+ curTblItr.next();
+ count++;
+ }
+ }
+ return count;
+ }
+}
diff --git a/hadoop-hdds/docs/content/_index.md b/hadoop-hdds/docs/content/_index.md
index f65c1527f47..fa50b0518aa 100644
--- a/hadoop-hdds/docs/content/_index.md
+++ b/hadoop-hdds/docs/content/_index.md
@@ -1,6 +1,6 @@
---
name: Ozone
-title: Overview
+title: An Introduction to Apache Ozone
menu: main
weight: -10
---
@@ -21,21 +21,98 @@ weight: -10
limitations under the License.
-->
-# Apache Ozone
+## What is Apache Ozone?
-{{<figure class="ozone-usage" src="/ozone-usage.png" width="60%">}}
+Apache Ozone is a scalable, distributed object store designed for lakehouse workloads,
+AI/ML, and cloud-native applications.
+Originating from the BigData analytics ecosystem, it handles both small and large files,
+supporting deployments up to billions of objects and exabytes of capacity.
+Ozone provides strong consistency guarantees,
+multiple protocol interfaces (including S3 compatibility), and configurable durability options.
-*_Ozone is a scalable, redundant, and distributed object store for Big data workloads. <p>
-Apart from scaling to billions of objects of varying sizes,
-Ozone can function effectively in containerized environments
-like Kubernetes._*
+## What it does?
-Applications like Apache Spark, Hive and YARN, work without any modifications when using Ozone. Ozone comes with a [Java client library]({{<ref "JavaApi.md">}}), [S3 protocol support]({{< ref "S3.md" >}}), and a [command line interface]({{< ref "Cli.md" >}}) which makes it easy to use Ozone.
+Ozone includes features relevant to large-scale storage requirements:
-Ozone consists of volumes, buckets, and keys:
+### Scale
-* Volumes are similar to user accounts. Only administrators can create or delete volumes.
-* Buckets are similar to directories. A bucket can contain any number of keys, but buckets cannot contain other buckets.
-* Keys are similar to files.
+Ozone's architecture separates metadata management from data storage. The Ozone Manager (OM) and Storage Container Manager (SCM) handle metadata operations, while Datanodes manage the physical storage of data blocks. This design allows for independent scaling of these components and supports incremental cluster growth.
-Check out the [Getting Started](start/) guide to dive right in and learn how to run Ozone on your machine or in the cloud.
+### Flexible Durability
+
+Ozone offers configurable data durability options per bucket or per object:
+* **Replication (RATIS):** Uses 3-way replication via the [Ratis (Raft)](https://ratis.apache.org) consensus protocol for high availability.
+* **Erasure Coding (EC):** Supports various EC codecs (e.g., Reed-Solomon) to reduce storage overhead compared to replication while maintaining specified durability levels.
+
+### Secure
+
+Security features are integrated at multiple layers:
+* **Authentication:** Supports Kerberos integration for user and service authentication.
+* **Authorization:** Provides Access Control Lists (ACLs) for managing permissions at the volume, bucket, and key levels. Supports Apache Ranger integration for centralized policy management.
+* **Encryption:** Supports TLS/SSL for data in transit and Transparent Data Encryption (TDE) for data at rest.
+* **Tokens:** Uses delegation tokens and block tokens for access control in distributed operations.
+
+### Performance
+
+Ozone's design considers performance for different access patterns:
+* **Throughput:** Intended for streaming reads and writes of large files. Data can be served directly from Datanodes after initial metadata lookup.
+* **Latency:** Metadata operations are managed by OM and SCM, designed for low-latency access.
+* **Small File Handling:** Includes mechanisms for managing metadata and storage for large quantities of small files.
+
+### Multiple Protocols
+
+Applications can access data stored in Ozone through several interfaces:
+* **S3 Protocol:** Provides an S3-compatible REST API, allowing use with S3-native applications and tools.
+* **Hadoop Compatible File System (OFS):** Offers the `ofs://` scheme for integration with Hadoop ecosystem tools (e.g., Iceberg, Spark, Hive, Flink, MapReduce).
+* **Native Java Client API:** A client library for Java applications.
+* **Command Line Interface (CLI):** Provides tools for administrative tasks and data interaction.
+
+### Efficient Storage Use
+
+Ozone includes features aimed at optimizing storage utilization:
+* **Erasure Coding:** Can reduce the physical storage footprint compared to 3x replication.
+* **Small File Handling:** Manages metadata and block allocation for small files.
+* **Containerization:** Groups data blocks into larger Storage Containers, which can simplify management and disk I/O.
+
+### Storage Management
+
+Ozone uses a hierarchical namespace and provides management tools:
+* **Namespace:** Organizes data into Volumes (often mapped to tenants) and Buckets (containers for objects), which hold Keys (objects/files).
+* **Quotas:** Administrators can set storage quotas at the Volume and Bucket levels.
+* **Snapshots:** Supports point-in-time, read-only snapshots of buckets for data protection and versioning.
+
+### Strong Consistency
+
+Ozone provides strong consistency for metadata and data operations. Reads reflect the results of the latest successfully completed write operations.
+
+## Key Characteristics
+
+The design of Ozone leads to certain characteristics relevant for large-scale data management:
+
+### Storage Costs
+
+Factors influencing storage costs include:
+* **Storage Efficiency:** Erasure Coding can reduce physical storage requirements.
+* **Hardware:** Designed to run on commodity hardware.
+* **Licensing:** Apache Ozone is open-source software under the Apache License 2.0.
+* **Scalability:** Clusters can be expanded by adding nodes or racks. Data rebalancing mechanisms help manage utilization.
+
+### Operations
+
+Aspects related to storage administration include:
+* **Unified Storage:** Can potentially serve as a common storage layer for different types of workloads.
+* **Management Tools:** Includes the Recon web UI for monitoring and CLI tools for administration.
+* **Maintenance:** Supports features like rolling upgrades, node decommissioning, and data balancing.
+
+### Hybrid Cloud Scenarios
+
+Ozone's S3 compatibility allows applications developed for S3 to run on-premises using Ozone. This can be relevant for hybrid cloud strategies or migrating workloads between on-premises and cloud environments.
+
+## Dive Deeper
+
+To learn more about Ozone, refer to the following sections:
+
+* **New to Ozone?** Try the **[Quick Start Guide]({{< ref "start" >}})** to set up a cluster.
+* **Want to understand the internals?** Read about the **[Core Concepts]({{< ref "concept" >}})** (architecture, replication, security).
+* **Need to use Ozone?** Check the **[User Guide]({{< ref "interface" >}})** for client interfaces and integrations.
+* **Managing a cluster?** Consult the **[Administrator Guide]({{< ref "tools" >}})** for installation, configuration, and operations.
diff --git a/hadoop-hdds/docs/content/concept/Comparison.md b/hadoop-hdds/docs/content/concept/Comparison.md
new file mode 100644
index 00000000000..a07aaa67251
--- /dev/null
+++ b/hadoop-hdds/docs/content/concept/Comparison.md
@@ -0,0 +1,77 @@
+---
+title: Comparison with Other Storage Technologies
+menu:
+ main:
+ parent: Architecture
+weight: 10
+---
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+This document provides a high-level comparison of Apache Ozone with other storage technologies.
+
+## Open Source Scale-out Storage Comparison
+
+Ozone is most often compared against other open source storage systems.
+
+| Tech | Type | Consistency | Scale | Big Data Integration | License | Notes |
+|---------------------|-----------------------|---------------------|--------------------------------------|----------------------|-----------------------|--------------------------------------------------------------------|
+| **Ozone** | Object / File | Strong | Exabyte scale, tens of billions keys | Native | Apache 2.0 | Modern Hadoop-native object store with S3 API |
+| **HDFS** | File | Strong | PBs, billions files | Native | Apache 2.0 | Classic Hadoop FS, no S3 API, tight Hadoop integration |
+| **Ceph** | Object / Block / File | Tunable / Eventual | Multi-PB, very large | Via S3 Gateway/CephFS | LGPLv2.1 / Ceph Foundation | General-purpose: underlying RADOS (object), RGW (object), CephFS (file) |
+| **MinIO** | Object | Strong | Petabyte scale | Via S3 connectors | AGPLv3 (SSPL-like) | Cloud-native S3 API, lightweight, fast, no FS semantics |
+| **Lustre** | Parallel File (POSIX) | Strong | PB scale, HPC | None | GPLv2 | HPC clusters, high-throughput parallel file system |
+| **GlusterFS** | File (POSIX) | Eventually consistent| Large, multi-PB | None | GPLv3 | General-purpose scale-out distributed file storage |
+| **OpenStack Swift** | Object | Eventual | Large, multi-PB | Via connectors | Apache 2.0 | S3-like multi-tenant object storage for private clouds |
+
+Ozone shines when users are in need of an Apache licensed, strongly consistent storage system that can scale to billions of keys/files and hundreds of PBs to EBs.
+
+## Proprietary Scale-Out Storage Comparison
+
+| Tech | Type | Consistency | Scale | Big Data Integration | Performance Focus | Notes |
+|-------------------------|------------------|-------------|----------------------|----------------------|--------------------------------------|----------------------------------------------------------------------|
+| **Isilon (Dell PowerScale)** | File (Scale-Out NAS) | Strong | PBs, billions of files | Indirect | High throughput, good mixed IO | Enterprise NAS, POSIX compliant, good for mixed workloads, backup, analytics |
+| **VAST** | File / Object | Strong | PBs | Yes, AI workloads | Ultra-low latency, all-flash NVMe | All-flash, NFS/S3, great for AI/ML and large unstructured datasets |
+| **WEKA** | Parallel File | Strong | PBs | HPC, AI | Ultra-low latency, high IOPS | High-performance file, GPU clusters, NFS/SMB/S3 |
+| **Spectrum Scale (GPFS)** | File (POSIX) | Strong | PBs | HPC, AI | High throughput, scale-out metadata | IBM, used in HPC/AI, policy tiering, good POSIX compliance |
+| **Scality** | Object | Strong | PBs | Some | Good throughput for large objects | Enterprise S3 API, multi-region, backup archives, hybrid cloud |
+| **Cloudian** | Object | Strong | PBs | Some | Good throughput for backup/archive | S3-compatible object storage, ransomware protection, hybrid cloud |
+
+
+The proprietary systems offer enterprise-grade quality, but they often require proprietary or certified hardware.
+Ozone shines when users look for commodity hardware, open systems and embrace the vibrant Apache big data open source community.
+
+## Cloud-Native Object Storage Comparison
+
+| Tech | Type | Consistency | Scale | Big Data Integration | Notes |
+|----------------|-----------------------|-------------|------------|----------------------|-------------------------------------------------------------------------|
+| **AWS S3** | Object | Strong | Exabyte+ | Native to cloud ecosystem | The de-facto standard for object storage; massive durability, S3 API leader |
+| **Azure ABFS** | File/Object (Data Lake Storage) | Strong | Exabyte+ | Azure-native | HDFS-like semantics for Spark/Hadoop; optimized for analytics |
+| **Google GCS** | Object | Strong | Exabyte+ | Native to cloud ecosystem | Globally distributed; strong consistency; well-integrated with BigQuery |
+| **OCI Object Storage** | Object | Strong | Exabyte+ | Via S3 API & native services | Oracle’s S3-compatible storage; integrates with OCI Data Flow |
+| **Alibaba OSS** | Object | Strong | Exabyte+ | Via S3 API & native services | S3-compatible, huge China/APAC footprint |
+| **IBM Cloud Object Storage** | Object | Strong | Exabyte+ | Via S3 API & native services | S3-compatible, geo-dispersed erasure coding for durability |
+
+These cloud storage offerings are only available from their respective public cloud vendors. In contrast, Ozone runs on-prem or in your private cloud, giving you full control.
+
+## Summary
+In summary, Ozone is the best fit in the following scenarios:
+
+1. Large on-prem big data clusters migrating from HDFS.
+2. You want S3 APIs but need strong Hadoop integration.
+3. You want to avoid vendor lock-in and grow cost-effectively on commodity hardware.
+4. You’re building a private or hybrid cloud with other open-source tools.
diff --git a/hadoop-hdds/docs/content/concept/OzoneManager.md b/hadoop-hdds/docs/content/concept/OzoneManager.md
index f58a4cd8f45..364abd53306 100644
--- a/hadoop-hdds/docs/content/concept/OzoneManager.md
+++ b/hadoop-hdds/docs/content/concept/OzoneManager.md
@@ -49,13 +49,13 @@ OM maintains a list of volumes, buckets, and keys.
For each user, it maintains a list of volumes.
For each volume, the list of buckets and for each bucket the list of keys.
-Ozone Manager will use Apache Ratis(A Raft protocol implementation) to
-replicate Ozone Manager state. This will ensure High Availability for Ozone.
+Ozone Manager will use [Apache Ratis](https://ratis.apache.org/)(A Raft protocol implementation) to
+replicate Ozone Manager state. This will ensure [High Availability for Ozone Manager]({{< ref "feature/OM-HA.md" >}}).
## Ozone Manager and Storage Container Manager
-The relationship between Ozone Manager and Storage Container Manager is best
+The relationship between Ozone Manager and [Storage Container Manager (SCM)]({{< ref "concept/StorageContainerManager.md" >}}) is best
understood if we trace what happens during a key write and key read.
### Key Write
@@ -68,12 +68,12 @@ Manager determines that you are allowed to write a key to the specified bucket,
OM needs to allocate a block for the client to write data.
* To allocate a block, Ozone Manager sends a request to Storage Container
-Manager (SCM); SCM is the manager of data nodes. SCM picks three data nodes
+Manager (SCM); SCM is the manager of [data nodes]({{< ref "concept/Datanodes.md" >}}). SCM picks three data nodes
into which client can write data. SCM allocates the block and returns the
-block ID to Ozone Manager.
+[block ID]({{< ref "concept/Datanodes.md" >}}) to Ozone Manager.
* Ozone manager records this block information in its metadata and returns the
-block and a block token (a security permission to write data to the block)
+block and a [block token]({{< ref "security/SecureOzone.md" >}}) (a security permission to write data to the block)
to the client.
* The client uses the block token to prove that it is allowed to write data to
@@ -108,12 +108,12 @@ Ozone provides a network service for the client and for administration commands.
* GetFileStatus, CreateDirectory, CreateFile, LookupFile
* ACL related
* Managing ACLs if [internal ACLs]({{< ref "security/SecurityAcls.md" >}}) are used instead of [Ranger]({{< ref "security/SecurityWithRanger.md" >}})
- * Delegation token (Get / Renew / Cancel)
+ * [Delegation token]({{< ref "security/SecureOzone.md" >}}) (Get / Renew / Cancel)
* For security
* Admin APIs
* Get S3 secret
* ServiceList (used for service discovery)
- * DBUpdates (used by [Recon]({{< ref "feature/Recon.md" >}}) downloads snapshots)
+ * DBUpdates (used by [Recon]({{< ref "concept/Recon.md" >}}) downloads snapshots)
### Persisted state
diff --git a/hadoop-hdds/docs/content/concept/StorageContainerManager.md b/hadoop-hdds/docs/content/concept/StorageContainerManager.md
index 48d509016a6..00aa7d0d38a 100644
--- a/hadoop-hdds/docs/content/concept/StorageContainerManager.md
+++ b/hadoop-hdds/docs/content/concept/StorageContainerManager.md
@@ -87,6 +87,27 @@ The following data is persisted in Storage Container Manager side in a specific
* Valid cert
* Used by the internal Certificate Authority to authorize other Ozone services
+## Safe Mode
+
+SCM (Storage Container Manager) enters safe mode on startup. This is a protective state that allows the system to become stable before it becomes fully operational. During safe mode, certain operations like block allocation are restricted.
+
+### How to Exit Safe Mode
+
+There are two ways to exit safe mode:
+
+1. **Automatic Exit:** SCM will automatically exit safe mode when a set of predefined `SafeModeExitRule`s are satisfied. These rules ensure that the cluster is in a healthy state. The primary rules are:
+ * **`DataNodeSafeModeRule`**: Checks if a minimum number of DataNodes have registered with the SCM. This is configured by `hdds.scm.safemode.min.datanode` (default: `3`).
+ * **`RatisContainerSafeModeRule`**: Checks if a certain percentage of containers with at least one replica reported are available. This is configured by `hdds.scm.safemode.threshold.pct` (default: `0.99`).
+ * **`HealthyPipelineSafeModeRule`**: Checks if a certain percentage of pipelines are healthy. This is configured by `hdds.scm.safemode.healthy.pipeline.pct` (default: `0.10`).
+ * **`OneReplicaPipelineSafeModeRule`**: Checks if a certain percentage of pipelines have at least one replica reported. This is configured by `hdds.scm.safemode.atleast.one.node.reported.pipeline.pct` (default: `0.90`).
+ * **`ECContainerSafeModeRule`**: Checks if a certain percentage of erasure coded block groups are healthy. This is also configured by `hdds.scm.safemode.threshold.pct` (default: `0.99`).
+
+2. **Manual Exit:** You can force SCM to exit safe mode using the `ozone admin safemode --force-exit` command.
+
+### Safe Mode Pre-Check
+
+There's also a "pre-check" phase. SCM will not exit safe mode until all pre-check rules are satisfied. The `DataNodeSafeModeRule` is a pre-check rule. This means that SCM will wait for a minimum number of DataNodes to be available before it even considers the other conditions for exiting safe mode.
+
## Notable configurations
key | default | description
diff --git a/hadoop-hdds/docs/content/feature/ContainerBalancer.md b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
new file mode 100644
index 00000000000..7faa99b0e1e
--- /dev/null
+++ b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
@@ -0,0 +1,108 @@
+---
+title: Container Balancer
+menu:
+ main:
+ parent: Features
+summary: How to use the Container Balancer in Ozone.
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+## Overview
+
+The Container Balancer is a tool in Apache Ozone that balances data containers across the cluster. Its primary goal is to ensure an even distribution of data based on disk space usage on datanodes. This helps to prevent some datanodes from becoming full while others remain underutilized.
+
+The balancer operates by moving `CLOSED` container replicas, which means it doesn't interfere with active I/O operations. It is designed to work with both regular and Erasure Coded (EC) containers. To maintain cluster stability, the Container Balancer's startup is delayed after a Storage Container Manager (SCM) failover.
+
+## Command Line Usage
+
+The Container Balancer is managed through the `ozone admin containerbalancer` command.
+
+### Start
+
+To start the Container Balancer with default settings:
+
+```bash
+ozone admin containerbalancer start
+```
+
+You can also start the balancer with specific options:
+
+```bash
+ozone admin containerbalancer start [options]
+```
+
+**Options:**
+
+| Option | Description |
+|-------------------------------------------------------| -------------------------------------------------------------------------------------------------------------------------------------- |
+| `-t`, `--threshold` | The percentage deviation from the average utilization of the cluster after which a datanode will be rebalanced. Default is 10%. |
+| `-i`, `--iterations` | The maximum number of consecutive iterations the balancer will run for. Default is 10. Use -1 for infinite iterations. |
+| `-d`, `--maxDatanodesPercentageToInvolvePerIteration` | The maximum percentage of healthy, in-service datanodes that can be involved in balancing in one iteration. Default is 20%. |
+| `-s`, `--maxSizeToMovePerIterationInGB` | The maximum size of data in GB to be moved in one iteration. Default is 500GB. |
+| `-e`, `--maxSizeEnteringTargetInGB` | The maximum size in GB that can enter a target datanode in one iteration. Default is 26GB. |
+| `-l`, `--maxSizeLeavingSourceInGB` | The maximum size in GB that can leave a source datanode in one iteration. Default is 26GB. |
+| `--balancing-iteration-interval-minutes` | The interval in minutes between each iteration of the Container Balancer. Default is 70 minutes. |
+| `--move-timeout-minutes` | The time in minutes to allow a single container to move from source to target. Default is 65 minutes. |
+| `--move-replication-timeout-minutes` | The time in minutes to allow a single container's replication from source to target as part of a container move. Default is 50 minutes. |
+| `--move-network-topology-enable` | Whether to consider network topology when selecting a target for a source. Default is false. |
+| `--include-datanodes` | A comma-separated list of datanode hostnames or IP addresses to be included in balancing. |
+| `--exclude-datanodes` | A comma-separated list of datanode hostnames or IP addresses to be excluded from balancing. |
+
+### Status
+
+To check the status of the Container Balancer:
+
+```bash
+ozone admin containerbalancer status
+```
+
+To get a more detailed status, including the history of iterations:
+
+```bash
+ozone admin containerbalancer status -v --history
+```
+
+### Stop
+
+To stop the Container Balancer:
+
+```bash
+ozone admin containerbalancer stop
+```
+
+## Configuration
+
+The Container Balancer can also be configured through the `ozone-site.xml` file.
+
+| Property | Description | Default Value |
+| ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | ------------- |
+| `hdds.container.balancer.utilization.threshold` | A cluster is considered balanced if for each datanode, the utilization of the datanode differs from the utilization of the cluster no more than this threshold. | 10% |
+| `hdds.container.balancer.datanodes.involved.max.percentage.per.iteration` | Maximum percentage of healthy, in-service datanodes that can be involved in balancing in one iteration. | 20% |
+| `hdds.container.balancer.size.moved.max.per.iteration` | The maximum size of data that will be moved by Container Balancer in one iteration. | 500GB |
+| `hdds.container.balancer.size.entering.target.max` | The maximum size that can enter a target datanode in each iteration. | 26GB |
+| `hdds.container.balancer.size.leaving.source.max` | The maximum size that can leave a source datanode in each iteration. | 26GB |
+| `hdds.container.balancer.iterations` | The number of iterations that Container Balancer will run for. | 10 |
+| `hdds.container.balancer.exclude.containers` | A comma-separated list of container IDs to exclude from balancing. | "" |
+| `hdds.container.balancer.move.timeout` | The amount of time to allow a single container to move from source to target. | 65m |
+| `hdds.container.balancer.move.replication.timeout` | The amount of time to allow a single container's replication from source to target as part of a container move. | 50m |
+| `hdds.container.balancer.balancing.iteration.interval` | The interval period between each iteration of Container Balancer. | 70m |
+| `hdds.container.balancer.include.datanodes` | A comma-separated list of Datanode hostnames or IP addresses. Only the Datanodes specified in this list are balanced. | "" |
+| `hdds.container.balancer.exclude.datanodes` | A comma-separated list of Datanode hostnames or IP addresses. The Datanodes specified in this list are excluded from balancing. | "" |
+| `hdds.container.balancer.move.networkTopology.enable` | Whether to take network topology into account when selecting a target for a source. | false |
+| `hdds.container.balancer.trigger.du.before.move.enable`| Whether to send a command to all healthy and in-service data nodes to run `du` immediately before starting a balance iteration. | false |
diff --git a/hadoop-hdds/docs/content/feature/Maintenance.md b/hadoop-hdds/docs/content/feature/Maintenance.md
new file mode 100644
index 00000000000..a1f0a6e3272
--- /dev/null
+++ b/hadoop-hdds/docs/content/feature/Maintenance.md
@@ -0,0 +1,78 @@
+---
+title: "Maintenance Mode"
+menu:
+ main:
+ parent: Features
+summary: Maintenance mode for Datanodes.
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# DataNode Maintenance Mode
+
+Maintenance mode is a feature in Apache Ozone that allows you to temporarily take a DataNode offline for maintenance operations (e.g., hardware upgrades, software updates) without triggering immediate data replication. Unlike decommissioning, which aims to permanently remove a DataNode and its data from the cluster, maintenance mode is designed for temporary outages.
+
+While in maintenance mode, a DataNode does not accept new writes but may still serve reads, assuming containers are healthy and online. Existing data on the DataNode will remain in place, and replication of its data will only be triggered if the DataNode remains in maintenance mode beyond a configurable timeout period. This allows for planned downtime without unnecessary data movement, reducing network overhead and cluster load.
+
+The DataNode transitions through the following operational states during maintenance:
+
+1. **IN_SERVICE**: The DataNode is fully operational and participating in data writes and reads.
+2. **ENTERING_MAINTENANCE**: The DataNode is transitioning into maintenance mode. New writes will be avoided.
+3. **IN_MAINTENANCE**: The DataNode is in maintenance mode. Data will not be written to it. If the DataNode remains in this state beyond the configured maintenance window, its data will start to be replicated to other DataNodes to ensure data durability.
+
+## Command Line Usage
+
+To place a DataNode into maintenance mode, use the `ozone admin datanode maintenance` command. You can specify a duration for the maintenance period. If no duration is specified, a default duration will be used (this can be configured).
+
+To check the current state of the datanodes, including their operational state, you can execute the following command:
+
+```shell
+ozone admin datanode list
+```
+
+To start maintenance mode for one or more DataNodes:
+
+```shell
+ozone admin datanode maintenance [-hV] [-id=<scmServiceId>] [--scm=<scm>] [--end=<hours>] [--force] [<hosts>...]
+```
+- `<hosts>`: A space-separated list of hostnames or IP addresses of the DataNodes to put into maintenance mode.
+- `--end=<hours>`: Optional. Automatically end maintenance after the given hours. By default, maintenance must be ended manually.
+- `--force`: Optional. Forcefully try to put the datanode(s) into maintenance mode.
+
+To take a DataNode out of maintenance mode and return it to `IN_SERVICE` state, you can use the `recommission` command:
+
+```shell
+ozone admin datanode recommission [-hV] [-id=<scmServiceId>] [--scm=<scm>] [<hosts>...]
+```
+
+## Configuration Properties
+
+The following properties, typically set in `ozone-site.xml`, are relevant to maintenance mode:
+
+- `hdds.scm.replication.maintenance.replica.minimum`: The minimum number of container replicas which must be available for a node to enter maintenance. Default value is `2`. If putting a node into maintenance reduces the available replicas for any container below this level, the node will remain in the `ENTERING_MAINTENANCE` state until a new replica is created.
+- `hdds.scm.replication.maintenance.remaining.redundancy`: The number of redundant containers in a group which must be available for a node to enter maintenance. Default value is `1`. If putting a node into maintenance reduces the redundancy below this value, the node will remain in the `ENTERING_MAINTENANCE` state until a new replica is created. For Ratis containers, the default value of 1 ensures at least two replicas are online, meaning 1 more can be lost without data becoming unavail [...]
+
+## Metrics
+
+The following SCM metrics are relevant to DataNode maintenance mode:
+
+- `DecommissioningMaintenanceNodesTotal`: This metric reports the total number of DataNodes that are currently in either decommissioning or maintenance mode.
+- `RecommissionNodesTotal`: This metric reports the total number of DataNodes that are currently being recommissioned (i.e., returning to `IN_SERVICE` state from either decommissioning or maintenance mode).
+
+## Reference
+
+For information on permanently removing a DataNode from the cluster, refer to the [Decommissioning](decommission.html) user documentation.
diff --git a/hadoop-hdds/docs/content/feature/OM-HA.md b/hadoop-hdds/docs/content/feature/OM-HA.md
index 12001bdfe7e..f055b295849 100644
--- a/hadoop-hdds/docs/content/feature/OM-HA.md
+++ b/hadoop-hdds/docs/content/feature/OM-HA.md
@@ -125,6 +125,84 @@ ozone om [global options (optional)] --bootstrap --force
Note that using the _force_ option during bootstrap could crash the OM process if it does not have updated configurations.
+## OM Leader Transfer
+
+The `ozone admin om transfer` command allows you to manually transfer the leadership of the Ozone Manager (OM) Raft group to a specific OM node or to a randomly chosen follower.
+
+### Usage
+
+```bash
+ozone admin om transfer -id <OM_SERVICE_ID> -n <NEW_LEADER_ID>
+ozone admin om transfer -id <OM_SERVICE_ID> -r
+```
+
+* `-id, --service-id`: Specifies the Ozone Manager Service ID.
+* `-n, --newLeaderId, --new-leader-id`: The node ID of the OM to which leadership will be transferred (e.g., `om1`).
+* `-r, --random`: Randomly chooses a follower to transfer leadership to.
+
+### Example
+
+To transfer leadership to `om2` in a cluster with service ID `cluster1`:
+
+```bash
+ozone admin om transfer -id cluster1 -n om2
+```
+
+To transfer leadership to a random follower:
+
+```bash
+ozone admin om transfer -id cluster1 -r
+```
+
+## OM Service Roles Listing
+
+The `ozone admin om roles` command lists all Ozone Managers and their respective Raft server roles (leader, follower, or candidate).
+
+### Usage
+
+```bash
+ozone admin om roles [-id <OM_SERVICE_ID>] [--json | --table]
+```
+
+* `-id, --service-id`: (Optional) Specifies the Ozone Manager Service ID.
+* `--json`: (Optional) Formats the output as JSON.
+* `--table`: (Optional) Formats the output as a table.
+
+### Example
+
+To list OM roles for `cluster1`:
+
+```bash
+ozone admin om roles -id cluster1
+```
+
+Example output:
+
+```
+om1 : LEADER (host1)
+om2 : FOLLOWER (host2)
+om3 : FOLLOWER (host3)
+```
+
+To list OM roles as a table:
+
+```bash
+ozone admin om roles -id cluster1 --table
+```
+
+Example table output:
+
+```
+Ozone Manager Roles
+-------------------
+Host Name | Node ID | Role
+-------------------
+host1 | om1 | LEADER
+host2 | om2 | FOLLOWER
+host3 | om3 | FOLLOWER
+-------------------
+```
+
## Automatic Snapshot Installation for Stale Ozone Managers
Sometimes an OM follower node may be offline or fall far behind the OM leader's raft log.
diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.md b/hadoop-hdds/docs/content/feature/SCM-HA.md
index 2b6ee72b7cf..7f9396fafe6 100644
--- a/hadoop-hdds/docs/content/feature/SCM-HA.md
+++ b/hadoop-hdds/docs/content/feature/SCM-HA.md
@@ -39,7 +39,7 @@ This logical name is called `serviceId` and can be configured in the `ozone-site
Most of the time you need to set only the values of your current cluster:
- ```XML
+ ```xml
<property>
<name>ozone.scm.service.ids</name>
<value>cluster1</value>
@@ -48,7 +48,7 @@ Most of the time you need to set only the values of your current cluster:
For each of the defined `serviceId` a logical configuration name should be defined for each of the servers
-```XML
+```xml
<property>
<name>ozone.scm.nodes.cluster1</name>
<value>scm1,scm2,scm3</value>
@@ -57,7 +57,7 @@ For each of the defined `serviceId` a logical configuration name should be defin
The defined prefixes can be used to define the address of each of the SCM services:
-```XML
+```xml
<property>
<name>ozone.scm.address.cluster1.scm1</name>
<value>host1</value>
@@ -90,6 +90,85 @@ ozone scm --bootstrap
Note: both commands perform one-time initialization. SCM still needs to be started by running `ozone --daemon start scm`.
+## SCM Leader Transfer
+
+The `ozone admin scm transfer` command allows you to manually transfer the leadership of the Storage Container Manager (SCM) Raft group to a specific SCM node or to a randomly chosen follower.
+
+Be aware of the node's status(eg. Safemode, Operational status), ozone currently has no ability to check the target node's status before transfering the leadership.
+
+### Usage
+
+```bash
+ozone admin scm transfer -id <SCM_SERVICE_ID> -n <NEW_LEADER_ID>
+ozone admin scm transfer -id <SCM_SERVICE_ID> -r
+```
+
+* `-id, --service-id`: Specifies the SCM Service ID.
+* `-n, --newLeaderId, --new-leader-id`: The SCM UUID (Raft peer ID) of the SCM to which leadership will be transferred (e.g., `e6877ce5-56cd-4f0b-ad60-4c8ef9000882`).
+* `-r, --random`: Randomly chooses a follower to transfer leadership to.
+
+### Example
+
+To transfer leadership to a specific SCM in a cluster with service ID `cluster1`:
+
+```bash
+ozone admin scm transfer -id cluster1 -n e6877ce5-56cd-4f0b-ad60-4c8ef9000882
+```
+
+To transfer leadership to a random follower:
+
+```bash
+ozone admin scm transfer -id cluster1 -r
+```
+
+## SCM Service Roles Listing
+
+The `ozone admin scm roles` command lists all Storage Container Managers and their respective Raft server roles (leader, follower, or candidate).
+
+### Usage
+
+```bash
+ozone admin scm roles [--json | --table]
+```
+
+* `--json`: (Optional) Formats the output as JSON.
+* `--table`: (Optional) Formats the output as a table.
+
+### Example
+
+To list SCM roles:
+
+```bash
+ozone admin scm roles
+```
+
+Example output:
+
+```
+host1:9876:LEADER:e6877ce5-56cd-4f0b-ad60-4c8ef9000882:192.168.1.1
+host2:9876:FOLLOWER:f2ba1c28-486a-4192-9a5f-b3752fe27d6b:192.168.1.2
+host3:9876:FOLLOWER:9c8d2b3a-e4f5-6789-0123-456789abcdef:192.168.1.3
+```
+
+To list SCM roles as a table:
+
+```bash
+ozone admin scm roles --table
+```
+
+Example table output:
+
+```
+Storage Container Manager Roles
+------------------------------------------------------------------------------------
+Host Name | Ratis Port | Role | Node ID | Host Address
+------------------------------------------------------------------------------------
+host1 | 9876 | LEADER | e6877ce5-56cd-4f0b-ad60-4c8ef9000882 | 192.168.1.1
+host2 | 9876 | FOLLOWER | f2ba1c28-486a-4192-9a5f-b3752fe27d6b | 192.168.1.2
+host3 | 9876 | FOLLOWER | 9c8d2b3a-e4f5-6789-0123-456789abcdef | 192.168.1.3
+------------------------------------------------------------------------------------
+```
+
## Auto-bootstrap
In some environments (e.g. Kubernetes) we need to have a common, unified way to initialize SCM HA quorum. As a reminder, the standard initialization flow is the following:
@@ -99,7 +178,7 @@ In some environments (e.g. Kubernetes) we need to have a common, unified way to
This can be improved: primordial SCM can be configured by setting `ozone.scm.primordial.node.id` in the config to one of the nodes.
-```XML
+```xml
<property>
<name>ozone.scm.primordial.node.id</name>
<value>scm1</value>
@@ -136,14 +215,14 @@ Sub-CA on the SCM’s are used to issue signed certificates for OM/DN in the clu
### How to enable security:
-```XML
+```xml
<property>
-<config>ozone.security.enable</config>
+<name>ozone.security.enable</name>
<value>true</value>
</property>
<property>
-<config>hdds.grpc.tls.enabled</config>
+<name>hdds.grpc.tls.enabled</name>
<value>true</value>
</property>
```
diff --git a/hadoop-hdds/docs/content/feature/om-bootstrapping-with-snapshots.md b/hadoop-hdds/docs/content/feature/om-bootstrapping-with-snapshots.md
new file mode 100644
index 00000000000..b1d7cfa2f60
--- /dev/null
+++ b/hadoop-hdds/docs/content/feature/om-bootstrapping-with-snapshots.md
@@ -0,0 +1,148 @@
+---
+title: OM Bootstrapping with Snapshots
+summary: Design for the OM bootstrapping process that is snapshot aware
+date: 2025-08-05
+jira: HDDS-12090
+status: implemented
+author: Swaminathan Balachandran
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# OM Bootstrapping with Snapshots
+
+# Problem Statement:
+
+The current bootstrapping mechanism for OM has inconsistencies when dealing with Snapshotted OM RocksDBs. Bootstrapping occurs without locking mechanisms, and active transactions may still modify snapshots RocksDB during the process. This can lead to a corrupted RocksDB instance on the follower OM post-bootstrapping. To resolve this, the bootstrapping process must operate on a consistent system state.
+
+Jira Ticket : [https://issues.apache.org/jira/browse/HDDS-12090](https://issues.apache.org/jira/browse/HDDS-12090)
+
+
+
+
+
+# Background on Snapshots
+
+## Snapshot Operations
+
+When a snapshot is taken on an Ozone bucket, the following steps occur:
+
+1. A RocksDB checkpoint of the active `om.db` is created.
+2. Deleted entries are removed from the `deletedKeyTable` and `deletedDirTable` in the Active Object Store (AOS) RocksDB. This is to just prevent the blocks from getting purged without checking for the key’s presence in the correct snapshot in the snapshot chain.
+3. A new entry is added to the `snapshotInfoTable` in the AOS RocksDB.
+
+## Current Bootstrap Model:
+
+The current model involves the follower OM initiating an HTTP request to the leader OM, which provides a consistent view of its state. Before bucket snapshots were introduced, this process relied solely on an AOS RocksDB checkpoint. However, with snapshots, multiple RocksDB instances (AOS RocksDB \+ snapshot RocksDBs) must be handled, complicating the process.
+
+### Workflow:
+
+* Follower Initiation:
+ Sends an exclude list of files already copied in previous batches.
+* Leader Actions:
+ * Creates an AOS RocksDB checkpoint.
+ * Performs a directory walk through:
+ * AOS RocksDB checkpoint directory.
+ * Snapshot RocksDB directories.
+ * Backup SST file directory (compaction backup directory).
+ * Identifies unique files to be copied in the next batch.
+ * Transfers files in batches, recreating hardlinks on the follower side as needed.
+
+### Issues with the Current Model
+
+1. Active transactions during bootstrapping may modify snapshot RocksDBs, leading to inconsistencies.
+2. Partial data copies can occur when double-buffer flushes or other snapshot-related operations are in progress.
+3. Large snapshot data sizes (often in GBs) require multi-batch transfers, increasing the risk of data corruption.
+
+# Proposed Fixes
+
+## Locking the Snapshot Cache
+
+Snapshot Cache is the class which is responsible for maintaining all rocksdb handles corresponding to a snapshot. The rocksdb handles are closed by the snapshot cache are closed from time to time if there are no references of the rocksdb being used by any of the threads in the system. Hence any operation on a snapshot would go through the snapshot cache increasing the reference count of that snapshot. Implementing a lock for this snapshot cache would prevent any newer threads from reques [...]
+
+With the above implementation of a lock there is a way to get a consistent snapshot of the entire OM. Now lets dive into various approaches to overall bootstrap flow.
+
+## Approach 1(Batching files over multiple tarballs):
+
+This approach builds on the current model by introducing size thresholds to manage locks and data transfers more efficiently.
+
+### Workflow:
+
+1. Follower Initiation:
+ * Sends an exclude list of previously copied files (identified by `inodeId`).
+2. Leader Directory Walk:
+ * Walks through AOS RocksDB, snapshot RocksDBs, and backup SST directories to identify files to transfer.
+ * Compares against the exclude list to avoid duplicate transfers.
+3. If the total size of files to be copied is more than **ozone.om.ratis.snapshot.lock.max.total.size.threshold** then the files would be directly sent over the stream as a tarball where the name of the files is the inodeId of the file.
+4. If the total size of files to be copied is less than equal to **ozone.om.ratis.snapshot.lock.max.total.size.threshold** then the snapshot cache lock is taken after waiting for the snapshot cache to completely get empty(No snapshot rocksdb should be open). Under the lock following operations would be performed:
+ * Take the AOS rocksdb checkpoint.
+ * A complete directory walk is done on AOS checkpoint rocksdb directory \+ all the snapshot rocksdb directories \+ backup sst file directory(compaction log directory) to figure out all the files to be copied and any file already present in the exclude list would be excluded.
+ * These files are added to the tarball where again the name of the file would be the inodeId of the file.
+5. As the files are being iterated the path of each file and their corresponding inodeIds would be tracked. When it is the last batch this map would also be written as a text file in the final tarball to recreate all the hardlinks on the follower node.
+
+The only drawback with this approach is that we might end up sending more data over the network because some sst files sent over the network could have been replaced because of compaction running concurrently on the active object store. But at the same time since the entire bootstrap operation is supposed to finish in the order of a few minutes, the amount of extra data would be really minimal assuming we could utmost write 30 MBs of data assuming there are 30000 keys written in 2 mins e [...]
+
+## Approach 1.1:
+
+This approach builds on the approach1 where along with introducing size thresholds under locks manage locks, we only rely on the number of files changed under the snapshot directory as the threshold.
+
+### Workflow:
+
+1. Follower Initiation:
+ * Sends an exclude list of previously copied files (identified by `inodeId`).
+2. Leader Directory Walk:
+ * Walks through AOS RocksDB, snapshot RocksDBs, and backup SST directories to identify files to transfer.
+ * Compares against the exclude list to avoid duplicate transfers.
+3. If either the total size to be copied or the total number of files under the snapshot rocksdb directory to be copied is more than **ozone.om.ratis.snapshot.max.total.sst.size** respectively then the files would be directly sent over the stream as a tarball where the name of the files is the inodeId of the file.
+4. If the total number of file size to be copied under the snapshot rocksdb directory is less than equal to **ozone.om.ratis.snapshot.max.total.sst.size** then the snapshot cache lock is taken after waiting for the snapshot cache to completely get empty(No snapshot rocksdb should be open). Under the lock following operations would be performed:
+ * Take the AOS rocksdb checkpoint.
+ * A complete directory walk is done on all the snapshot rocksdb directories to figure out all the files to be copied and any file already present in the exclude list would be excluded.
+ * Hard links of these files are added to tmp directory on the leader.
+ * Exit lock
+ * After the lock all files under the tmp directory, AOS Rocksdb checkpoint directory and compaction backup directory have to be written to the tarball. As the files are being iterated the path of each file and their corresponding inodeIds would be tracked. Since this is the last batch this map would also be written as a text file in the final tarball to recreate all the hardlinks on the follower node.
+
+The drawbacks for this approach is the same as approach 1, but here we are optimizing on the amount of time lock is held by performing very lightweight operations under the lock. So this is a more optimal approach since it minimises the lock wait time on other threads.
+
+## Approach 2(Single tarball creation under lock):
+
+The approach 2 here proposes to create a single tarball file on the disk and stream the chunks of the tarball over multiple http batch request from the follower.
+Following is the flow for creating the tarball:
+
+1. Snapshot cache lock is taken after waiting for the snapshot cache to become completely empty(No snapshot rocksdb should be open). Under the lock following operations would be performed:
+ * Take the AOS rocksdb checkpoint.
+ * A complete directory walk is done on AOS rocksdb directory \+ all the snapshot rocksdb directories \+ backup sst file directory(compaction log directory) to figure out all the files to be copied to create a single tarball.
+2. This tarball should be streamed batch by batch to the follower in a paginated fashion.
+
+The drawback with this approach is that the double buffer would be blocked for a really long time if there is a lot of data to be tarballed. If the total snapshot size of the OM dbs put together is 1 TB, considering the tarball writes go to an NVMe and considering the write throughput for an NVMe drive is around 5 GB/sec then the tarball write might take a total of 1024/5 secs \= 3 mins. Blocking the double buffer thread for 3 mins seems to be a bad idea, but at the same time this would [...]
+
+## Approach 3(Creating a checkpoint of the snapshot rocksdb under lock):
+
+The approach 3 here proposes to create a rocksdb checkpoint for each and every snapshot rocksdb in the system along with the AOS rocksdb under the snapshot cache lock. Outside of the lock we could either create a single tarball file as done in approach 2 or stream the files in batches as multiple tarball file similar to approach 1 to the follower.
+Following is the flow for creating the tarball:
+
+1. Snapshot cache lock is taken after waiting for the snapshot cache to become completely empty(No snapshot rocksdb should be open). Under the lock following operations would be performed:
+ * Take the AOS rocksdb checkpoint.
+ * Take rocksdb checkpoint of each and every snapshot in the system by iterating through the snapshotInfo table of AOS checkpoint rocksdb.
+2. Now the files in the checkpoint directories have to be streamed to the follower as done in either approach 1 or approach 2\.
+
+The drawback with this approach is that this would double the number of hardlinks in the file system which could have potential impact on performance during bootstrap, considering the case in systems where the total number of files and hardlinks in the system order up to 5 million files.
+
+## Recommendations
+
+Approach 1 is the most optimized solution as it balances the amount of time under the lock by minimising the amount of IO ops inside the lock by introducing another threshold config to track this. Moreover taking this approach will also need the most minimal amount of code change as it doesn’t differ from the current approach by much. While approach 2 might look simpler but this would imply revamping the entire bootstrap logic currently in place and moreover this approach might increase [...]
+
+
+Final approach implemented is the Approach 1.1
+
diff --git a/hadoop-hdds/docs/content/integration/DistCp.md b/hadoop-hdds/docs/content/integration/DistCp.md
index 9d8709f0edc..b2ab003ab18 100644
--- a/hadoop-hdds/docs/content/integration/DistCp.md
+++ b/hadoop-hdds/docs/content/integration/DistCp.md
@@ -86,6 +86,8 @@ By specifying the appropriate checksum configuration or skipping the validation,
When data resides in an HDFS encryption zone or Ozone encrypted buckets, the file checksum will not match. This is because the underlying block data differs due to the use of a new EDEK (Encryption Data Encryption Key) at the destination. In such cases, specify the `-skipcrccheck` parameter to avoid job failures.
+Note that `distcp` is supported between encrypted Ozone clusters starting with Ozone 2.0.
+
For more information about using Hadoop DistCp, consult the [DistCp Guide](https://hadoop.apache.org/docs/current/hadoop-distcp/DistCp.html).
## Troubleshooting Common Issues
diff --git a/hadoop-hdds/docs/content/interface/Ofs.md b/hadoop-hdds/docs/content/interface/Ofs.md
index d37e2421d0c..37354a7ccd4 100644
--- a/hadoop-hdds/docs/content/interface/Ofs.md
+++ b/hadoop-hdds/docs/content/interface/Ofs.md
@@ -273,3 +273,161 @@ This feature wouldn't degrade server performance as the loop is on the client.
Think it as a client is issuing multiple requests to the server to get all the
information.
+## Migrating from HDFS
+
+This guide helps you migrate applications from HDFS to Ozone by detailing the API-level compatibility of the Ozone File System (Ofs).
+
+To ensure a smooth transition, you should first verify that your existing applications will work with Ofs. You can check for potential incompatibilities by identifying the HDFS APIs your applications use.
+
+* **For Cluster Administrators**: Analyze the NameNode audit logs on your source HDFS cluster to identify the operations performed by your applications.
+* **For Application Developers**: Review your application's source code to identify which `FileSystem` or `DistributedFileSystem` APIs are being called.
+
+Once you have a list of APIs, compare it against the tables below to identify any unsupported operations.
+
+### Supported FileSystem APIs
+
+The following standard `FileSystem` APIs are supported by Ofs.
+
+| Operation | NameNode audit log | Description | Support |
+|-------------------------|---------------------------|-----------------------------------------------------|---------------|
+| `access` | `checkAccess` | Checks if the user can access a path. | Supported |
+| `create` | `create` | Creates a new file. | Supported |
+| `open` | `open` | Opens a file for reading. | Supported |
+| `rename` | `rename` | Renames a file or directory. | Supported [1] |
+| `delete` | `delete` | Deletes a file or directory. | Supported [2] |
+| `listStatus` | `listStatus` | Lists the status of files in a directory. | Supported [3] |
+| `mkdirs` | `mkdirs` | Creates a directory and its parents. | Supported |
+| `getFileStatus` | `getfileinfo` | Gets the status of a file. | Supported |
+| `setTimes` | `setTimes` | Sets the modification and access times. | Supported |
+| `getLinkTarget` | `getfileinfo` | Gets the target of a symbolic link. | Supported [4] |
+| `getFileChecksum` | `open` | Gets the checksum of a file. | Supported |
+| `setSafeMode` | `safemode_leave`, `safemode_enter`, `safemode_get`, `safemode_force_exit` | Enters or leaves safe mode. | Supported |
+| `recoverLease` | `recoverLease` | Recovers a file lease. | Supported |
+| `isFileClosed` | `isFileClosed` | Checks if a file is closed. | Supported |
+| `createSnapshot` | `createSnapshot` | Creates a snapshot. | Supported [5] |
+| `deleteSnapshot` | `deleteSnapshot` | Deletes a snapshot. | Supported [5] |
+| `renameSnapshot` | `renameSnapshot` | Renames a snapshot. | Supported [5] |
+| `getSnapshotDiffReport` | `computeSnapshotDiff` | Gets a snapshot diff report. | Supported [5] |
+| `getContentSummary` | `contentSummary` | Gets the content summary of a path. | Supported |
+| `getDelegationToken` | `getDelegationToken` | Gets a delegation token. | Supported |
+| `globStatus` | `listStatus` | Finds files matching a pattern. | Supported |
+| `copyFromLocalFile` | `getfileinfo` | Copies a file from the local filesystem. | Supported |
+| `exists` | `getfileinfo` | Checks if a path exists. | Supported |
+| `getFileBlockLocations` | `open` | Gets file block locations. | Supported |
+| `getTrashRoot` | `listSnapshottableDirectory`, `getEZForPath` | Gets the trash root for a path. | Supported |
+| `getTrashRoots` | `listStatus`, `listEncryptionZones` | Gets all trash roots. | Supported |
+| `isDirectory` | `getfileinfo` | Checks if a path is a directory. | Supported |
+| `isFile` | `getfileinfo` | Checks if a path is a file. | Supported |
+| `listFiles` | `listStatus` | Returns a remote iterator for files. | Supported |
+| `listLocatedStatus` | `listStatus` | Returns a remote iterator for located file statuses.| Supported |
+| `listStatusIterator` | `listStatus` | Returns a remote iterator for file statuses. | Supported |
+| `getDefaultBlockSize` | N/A | Gets the default block size. | Supported |
+| `getDefaultReplication` | N/A | Gets the default replication factor. | Supported |
+| `getHomeDirectory` | N/A | Gets the user's home directory. | Supported |
+| `getServerDefaults` | N/A | Gets the server default values. | Supported |
+| `getWorkingDirectory` | N/A | Gets the current working directory. | Supported |
+| `hasPathCapability` | N/A | Queries for a path capability. | Supported |
+| `setWorkingDirectory` | N/A | Sets the current working directory. | Supported |
+| `supportsSymlinks` | N/A | Checks if symbolic links are supported. | Supported |
+
+*Note: An audit log of N/A means the API is client-side only and does not access the NameNode.*
+
+### Unsupported FileSystem APIs
+
+The following standard `FileSystem` APIs are not supported by Ofs.
+
+| Operation | NameNode audit log | Description |
+|--------------------|----------------------|--------------------------------|
+| `append` | `append` | Appends to an existing file. |
+| `setPermission` | `setPermission` | Sets the permission of a file. |
+| `setOwner` | `setOwner` | Sets the owner of a file. |
+| `setReplication` | `setReplication` | Sets the replication factor. |
+| `createSymlink` | `createSymlink` | Creates a symbolic link. |
+| `resolveLink` | `getfileinfo` | Resolves a symbolic link. |
+| `setXAttr` | `setXAttr` | Sets an extended attribute. |
+| `getXAttr` | `getXAttrs` | Gets an extended attribute. |
+| `getXAttrs` | `getXAttrs` | Gets extended attributes. |
+| `listXAttrs` | `listXAttrs` | Lists extended attributes. |
+| `removeXAttr` | `removeXAttr` | Removes an extended attribute. |
+| `setAcl` | `setAcl` | Sets an ACL. |
+| `getAclStatus` | `getAclStatus` | Gets an ACL status. |
+| `modifyAclEntries` | `modifyAclEntries` | Modifies ACL entries. |
+| `removeAclEntries` | `removeAclEntries` | Removes ACL entries. |
+| `removeDefaultAcl` | `removeDefaultAcl` | Removes the default ACL. |
+| `removeAcl` | `removeAcl` | Removes an ACL. |
+| `truncate` | `truncate` | Truncates a file. |
+| `concat` | `concat` | Concatenates files. |
+| `listCorruptFileBlocks` | `listCorruptFileBlocks` | List corrupted file blocks |
+| `msync` | N/A | Does not have a corresponding HDFS audit log. |
+
+### Unsupported HDFS-Specific APIs
+
+The following APIs are specific to HDFS's `DistributedFileSystem` implementation and are not part of the generic `org.apache.hadoop.fs.FileSystem` API. Therefore, they are not supported by Ofs. However, see the footnotes below for equivalent APIs.
+
+| Operation | NameNode audit log | Description | Support |
+|--------------------------------|------------------------------|-----------------------------------------------------|----------------|
+| `getErasureCodingPolicy` | `getErasureCodingPolicy` | Gets the erasure coding policy of a file/dir. | Unsupported [6]|
+| `setErasureCodingPolicy` | `setErasureCodingPolicy` | Sets an erasure coding policy on a directory. | Unsupported [6]|
+| `unsetErasureCodingPolicy` | `unsetErasureCodingPolicy` | Unsets an erasure coding policy on a directory. | Unsupported [6]|
+| `addErasureCodingPolicies` | `addErasureCodingPolicies` | Adds erasure coding policies. | Unsupported [6]|
+| `getErasureCodingPolicies` | `getErasureCodingPolicies` | Gets the available erasure coding policies. | Unsupported [6]|
+| `removeErasureCodingPolicy` | `removeErasureCodingPolicy` | Removes an erasure coding policy. | Unsupported [6]|
+| `enableErasureCodingPolicy` | `enableErasureCodingPolicy` | Enables an erasure coding policy. | Unsupported [6]|
+| `disableErasureCodingPolicy` | `disableErasureCodingPolicy` | Disables an erasure coding policy. | Unsupported [6]|
+| `getErasureCodingCodecs` | `getErasureCodingCodecs` | Lists all erasure coding codecs. | Unsupported [6]|
+| `getECTopologyResultForPolicies` | `getECTopologyResultForPolicies` | Get erasure coding topology result for policies. | Unsupported [6]|
+| `getSnapshotListing` | `ListSnapshot` | List all snapshots of a snapshottable directory. | Unsupported [6]|
+| `allowSnapshot` | `allowSnapshot` | Allows snapshots to be taken on a directory. | Unsupported [6]|
+| `disallowSnapshot` | `disallowSnapshot` | Disallows snapshots to be taken on a directory. | Unsupported [6]|
+| `provisionSnapshotTrash` | `getfileinfo`, `mkdirs`, `setPermission` | Provision trash for a snapshottable directory. | Unsupported [6]|
+| `createEncryptionZone` | `createEncryptionZone` | Creates an encryption zone. | Unsupported [6]|
+| `getEZForPath` | `getEZForPath` | Gets the encryption zone for a path. | Unsupported [6]|
+| `listEncryptionZones` | `listEncryptionZones` | Lists all encryption zones. | Unsupported [6]|
+| `reencryptEncryptionZone` | `reencryptEncryptionZone` | Reencrypt an encryption zone. | Unsupported [6]|
+| `listReencryptionStatus` | `listReencryptionStatus` | List reencryption status. | Unsupported [6]|
+| `getFileEncryptionInfo` | `getfileinfo` | Get file encryption info. | Unsupported [6]|
+| `provisionEZTrash` | `getEZForPath`, `getfileinfo`, `mkdirs`, `setPermission` | Provision trash for an encryption zone. | Unsupported [6]|
+| `setQuota` | `clearQuota` or `clearSpaceQuota` or `setQuota` or `setSpaceQuota` | Sets the quota usage for a path. | Unsupported [6]|
+| `getQuotaUsage` | `quotaUsage` | Gets the quota usage for a path. | Unsupported [6]|
+| `setQuotaByStorageType` | `setSpaceQuota` | Sets quota by storage type for a path. | Unsupported [6]|
+| `unsetStoragePolicy` | `unsetStoragePolicy` | Unsets a storage policy on a file or directory. | Unsupported |
+| `setStoragePolicy` | `setStoragePolicy` | Sets a storage policy on a file or directory. | Unsupported |
+| `getStoragePolicy` | `getStoragePolicy` | Gets the storage policy of a file or directory. | Unsupported |
+| `satisfyStoragePolicy` | `satisfyStoragePolicy` | Satisfies the storage policy of a file. | Unsupported |
+| `addCachePool` | `addCachePool` | Adds a cache pool. | Unsupported |
+| `modifyCachePool` | `modifyCachePool` | Modifies a cache pool. | Unsupported |
+| `removeCachePool` | `removeCachePool` | Removes a cache pool. | Unsupported |
+| `listCachePools` | `listCachePools` | Lists all cache pools. | Unsupported |
+| `addCacheDirective` | `addCacheDirective` | Adds a cache directive. | Unsupported |
+| `modifyCacheDirective` | `modifyCacheDirective` | Modifies a cache directive. | Unsupported |
+| `removeCacheDirective` | `removeCacheDirective` | Removes a cache directive. | Unsupported |
+| `listCacheDirectives` | `listCacheDirectives` | Lists cache directives. | Unsupported |
+| `getSlowDatanodeStats` | `datanodeReport` | Get slow datanode stats. | Unsupported |
+| `saveNamespace` | `saveNamespace` | Save the namespace. | Unsupported |
+| `restoreFailedStorage` | `checkRestoreFailedStorage`, `enableRestoreFailedStorage`, `disableRestoreFailedStorage` | Restore failed storage. | Unsupported |
+| `refreshNodes` | `refreshNodes` | Refresh nodes. | Unsupported |
+| `setBalancerBandwidth` | `setBalancerBandwidth` | Set balancer bandwidth. | Unsupported |
+| `metaSave` | `metaSave` | Meta save. | Unsupported |
+| `rollingUpgrade` | `queryRollingUpgrade`, `startRollingUpgrade`, `finalizeRollingUpgrade` | Rolling upgrade. | Unsupported |
+| `finalizeUpgrade` | `finalizeUpgrade` | Finalize upgrade. | Unsupported [7] |
+| `listOpenFiles` | `listOpenFiles` | List open files. | Unsupported [7] |
+
+**Footnotes:**
+
+[1] Renaming files or directories across different buckets is not supported. Within a File System Optimized (FSO) bucket, rename is an atomic metadata operation. For legacy buckets, renaming a directory is a non-atomic operation that renames each file and subdirectory individually.
+
+[2] Deleting the filesystem root is not allowed. Recursively deleting a volume is also not supported.
+
+[3] Recursive listing is not supported at the root or volume level.
+
+[4] Ofs supports "linked buckets," where one bucket serves as a reference to another. However, general-purpose symbolic links for files or directories are not supported.
+
+[5] Snapshots are supported at the bucket level only.
+
+[6] For operations related to encryption zones, erasure coding, snapshots, and quotas, use the corresponding Ozone bucket-level APIs instead.
+
+[7] Replace with `OzoneManagerProtocol.finalizeUpgrade()` and `OzoneManagerProtocol.listOpenFiles()`.
+
+*The following audit logs are typically produced by HDFS internal services and are not relevant for application migration: `slowDataNodesReport`, `getDatanodeStorageReport`, `rollEditLog`, `renewDelegationToken`, `cancelDelegationToken`, `gcDeletedSnapshot`.*
+*The following audit logs are produced by the HDFS output stream: `getAdditionalBlock`, `getAdditionalDatanode`, `abandonBlock`, `completeFile`, `fsync`.*
+*The `getPreferredBlockSize` audit log is used in testing only.*
diff --git a/hadoop-hdds/docs/content/interface/S3.md b/hadoop-hdds/docs/content/interface/S3.md
index 13ed1bc332a..3b3358309b4 100644
--- a/hadoop-hdds/docs/content/interface/S3.md
+++ b/hadoop-hdds/docs/content/interface/S3.md
@@ -56,34 +56,67 @@ By default it uses the path-style addressing. To use virtual host style URLs set
</property>
```
-## Implemented REST endpoints
+## Implemented REST Endpoints
-Operations on S3Gateway service:
+The Ozone S3 Gateway implements a substantial subset of the Amazon S3 REST API. The tables below summarize each API’s support status, its feature, and any relevant notes, including known deviations from AWS S3 behavior.
-Endpoint | Status |
-------------|-------------|
-GET service | implemented |
+### Bucket Operations
-Operations on Bucket:
+| **API Name** | **Feature** | **Note** |
+|---------------------------------------------------------------------------------------------------|-------------|----------|
+| ✅ [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html) | Lists all buckets owned by the authenticated user. | Returns the full bucket list. |
+| ✅ [CreateBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html) | Creates a new bucket. | **Non-compliant behavior:** The default bucket ACL may include extra group permissions instead of being strictly private. Bucket names must adhere to S3 naming conventions. |
+| ✅ [HeadBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html) | Checks for the existence of a bucket. | Returns a 200 status if the bucket exists. |
+| ✅ [DeleteBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html) | Deletes a bucket. | Bucket must be empty before deletion. |
+| ✅ [GetBucketLocation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html) | Retrieves the location (region) of a bucket. | Typically returns a default region (e.g., `us-east-1`), which may differ from AWS if region-specific responses are expected. |
-Endpoint | Status | Notes
-------------------------------------|-------------|---------------
-GET Bucket (List Objects) Version 2 | implemented |
-HEAD Bucket | implemented |
-DELETE Bucket | implemented |
-PUT Bucket (Create bucket) | implemented |
-Delete Multiple Objects (POST) | implemented |
+### Object Operations
-Operation on Objects:
+| **API Name** | **Feature** | **Note** |
+|-------------------------------------------------------------------------------------------|-------------|----------|
+| ✅ [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) | Retrieves the contents of an object. | **Non-compliant behavior:** For non-existent objects, Ozone may return a generic 404 without the structured XML error body defined by AWS S3. |
+| ✅ [HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) | Retrieves metadata for an object. | Similar to GET Object, error responses for missing objects might lack detailed AWS-style error XML. |
+| ✅ [PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) | Uploads a new object. | Supports both single PUT and multipart uploads. **Non-compliant behavior:** Any ACL headers provided in the request are ignored. |
+| ✅ [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html) | Deletes a single object. | Returns standard error codes for missing objects. |
+| ✅ [DeleteObjects](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) | Deletes multiple objects in one request. | Bulk delete operation with aggregated error reporting. |
+| ✅ [CopyObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html) | Copies an object (intra- or inter-bucket). | Supports metadata directives (COPY/REPLACE). **Non-compliant behavior:** Self-copying for metadata updates may differ slightly from AWS S3. |
+| ✅ [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) | Lists objects in a bucket with filtering and pagination. | Supports prefix, delimiter, and continuation tokens. |
-Endpoint | Status | Notes
-------------------------------------|-----------------|---------------
-PUT Object | implemented |
-GET Object | implemented |
-Multipart Upload | implemented | Except the listing of the current MultiPartUploads.
-DELETE Object | implemented |
-HEAD Object | implemented |
+### Object Tagging Operations
+| **API Name** | **Feature** | **Note** |
+|-------------------------------------------------------------------------------------------------------|-------------|----------|
+| ✅ [PutObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html) | Adds or updates tags on an object. | Fully supported; updates replace any existing tag set. |
+| ✅ [GetObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html) | Retrieves the tag set of an object. | Returns the tag set as defined by the S3 API. |
+| ✅ [DeleteObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html) | Removes the tag set from an object. | Clears all tags associated with the object. |
+
+### Multipart Upload Operations
+
+| **API Name** | **Feature** | **Note** |
+|--------------|-----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| ✅ [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html) | Starts a multipart upload session. | Returns an UploadId for subsequent parts. |
+| ✅ [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html) | Uploads an individual part for a multipart upload. | Part numbering and ETag computation follow AWS semantics. |
+| ✅ [ListMultipartUploads](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html) | Lists in-progress multipart uploads in a bucket. | Up to 1000 multipart uploads are returned in a batch by default. |
+| ✅ [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html) | Lists uploaded parts for an in-progress multipart upload. | Supports pagination if many parts exist. |
+| ✅ [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) | Assembles all uploaded parts into a final object. | Returns an ETag that follows AWS multipart rules. |
+| ✅ [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html) | Aborts an in-progress multipart upload. | Frees up storage used by uploaded parts. Incomplete multipart uploads older than 7 days are automatically aborted by Ozone. |
+
+### Additional Operations
+
+| **API Name** | **Feature** | **Note** |
+|--------------|-------------|----------|
+| ✅ Generate Presigned URL | Generates a temporary URL for accessing an object. | Uses AWS Signature V4. **Non-compliant behavior:** The generated URL may include a fixed default region rather than dynamically reflecting the bucket’s location. Currently, Ozone only supports generating presigned URLs for `GetObject`. Support for other operations is tracked in [HDDS-5195](https://issues.apache.org/jira/browse/HDDS-5195) and [HDDS-13393](https://issues.apache.org/jira/browse/HDDS-13393). |
+
+---
+
+> **Additional Notes:**
+> - **Advanced Features Not Supported:**
+> - **ACLs, Bucket Policies, CORS Configuration, and Website Hosting:** These are not fully implemented; Ozone uses an internal permission model that deviates from AWS S3.
+> - **Bucket Versioning, Object Locking, Server-Side Encryption, and S3 Select:** These features are currently not supported.
+> - **Conditional Requests:** Support for conditional requests (e.g., `If-Match`, `If-None-Match`) is planned and tracked in [HDDS-13117](https://issues.apache.org/jira/browse/HDDS-13117).
+> - **Lifecycle configuration, cross region replication, S3 event notifications** are being implemented and in the roadmap.
+> - While Ozone S3 Gateway provides extensive support for common S3 operations, users should be aware of the above non-compliant behaviors and limitations when integrating with applications expecting full AWS S3 functionality.
+> - While Ozone S3 Gateway does not support S3 Server-Side Encryption, it does support encrypted buckets using Apache Ranger KMS. For more information, see the [Transparent Data Encryption]({{< relref "security/SecuringTDE.md" >}}) documentation.
## Security
diff --git a/hadoop-hdds/docs/content/start/ProductionDeployment.md b/hadoop-hdds/docs/content/start/ProductionDeployment.md
new file mode 100644
index 00000000000..ed24a7b2671
--- /dev/null
+++ b/hadoop-hdds/docs/content/start/ProductionDeployment.md
@@ -0,0 +1,89 @@
+---
+title: Production Deployment
+weight: 6
+menu:
+ main:
+ parent: Getting Started
+---
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+This document provides guidance on the requirements and best practices for a production deployment of Apache Ozone.
+
+## Ozone Components
+
+A typical production Ozone cluster includes the following services:
+
+* **Ozone Manager (OM)**: Manages the namespace and metadata of the Ozone cluster. A production cluster requires 3 OM instances for high availability.
+* **Storage Container Manager (SCM)**: Manages the data nodes and pipelines. A production cluster requires 3 SCM instances for high availability.
+* **DataNode**: Stores the actual data in containers. A production cluster requires at least 3 DataNodes.
+* **Recon**: A web-based UI for monitoring and managing the Ozone cluster. A Recon server is strongly recommended, though not required.
+* **S3 Gateway (S3G)**: An S3-compatible gateway for accessing Ozone. Multiple S3 Gateway instances are strongly recommended to load balance S3 traffic.
+* **HttpFs**: An HDFS-compatible API for accessing Ozone. This is an optional component.
+
+## Requirements
+
+### System Requirements
+
+* **Hardware**: Bare metal machines are recommended for optimal performance. Virtual machines or containers are not recommended for production deployments.
+* **Operating System**: Linux (recommended distributions: Red Hat 8/Rocky 8+, Ubuntu, SUSE; supported architectures: x86/ARM).
+* **Java Development Kit (JDK)**: Version 8 or higher.
+* **Time Synchronization**: A time synchronization service such as Chrony or ntpd must be enabled to prevent time drift.
+
+### Memory Requirements
+
+* **Ozone Manager (OM), Storage Container Manager (SCM), and Recon**: Recommended heap size in large production clusters is 64GB.
+* **DataNode, S3 Gateway, and HttpFs**: Recommended heap size is 31GB.
+
+### Storage Requirements
+
+* **Ozone Manager (OM), Storage Container Manager (SCM), and Recon Metadata Storage**: Use SAS SSD or NVMe SSD for metadata (RocksDB and Ratis) to ensure optimal performance. It is recommended to use RAID 1 (disk mirroring) for the metadata disks to protect against disk failures.
+* **DataNode Storage**:
+ * **Ratis Log**: Use SAS SSD or NVMe SSD for the Ratis log directory for low latency writes.
+ * **Container Data**: Hard disks are acceptable for container data storage.
+ * **Disk Configuration**: It is recommended to use a JBOD (Just a Bunch Of Disks) configuration instead of RAID. Ozone is a replicated distributed storage system and handles data redundancy. Using RAID can decrease performance without providing additional data protection benefits.
+* **Storage Type**: Use direct-attached storage. Do not use Network Attached Storage (NAS) or Storage Area Network (SAN).
+
+### Network Requirements
+
+* **Network Bandwidth**: A minimum of 25Gbps network card bandwidth is recommended.
+* **Network Topology**: A leaf-spine network topology with an oversubscription ratio below 3:1 is recommended for predictable performance.
+
+### Security Requirements (Optional but Recommended)
+
+* **Kerberos**: A Kerberos environment, including a Key Distribution Center (KDC), is recommended for enhanced security.
+
+## Recommended Configurations
+
+### Linux Kernel
+
+* **CPU Governor**: Set the CPU scaling driver to `performance` mode to maximize performance.
+* **Transparent Hugepage**: Disable Transparent Hugepage to avoid performance issues.
+* **SELinux**: Disable SELinux.
+* **Swappiness**: Set `vm.swappiness=1` to minimize swapping.
+
+### Local File System
+
+* **LVM**: Disable Logical Volume Manager (LVM) for data drives.
+* **File System**: Use `ext4` or `xfs` file systems.
+* **Mount Options**: Mount drives with the `noatime` option to reduce unnecessary disk writes. For SSDs, also add the `discard` option.
+
+### Ozone Configuration
+
+* **Monitoring**: Install Prometheus and Grafana for monitoring the Ozone cluster. For audit logs, consider using a log ingestion framework such as the ELK Stack (Elasticsearch, Logstash, and Kibana) with FileBeat, or other similar frameworks. Alternatively, you can use Apache Ranger to manage audit logs.
+* **Pipeline Limits**: Increase the number of allowed write pipelines to better suit your workload by adjusting `ozone.scm.datanode.pipeline.limit` and `ozone.scm.ec.pipeline.minimum`.
+* **Heap Sizes**: Configure sufficient heap sizes for Ozone Manager (OM), Storage Container Manager (SCM), Recon, DataNode, S3 Gateway (S3G), and HttpFs services to ensure stability.
diff --git a/hadoop-hdds/docs/content/start/ProductionDeployment.zh.md b/hadoop-hdds/docs/content/start/ProductionDeployment.zh.md
new file mode 100644
index 00000000000..4620ccf31bc
--- /dev/null
+++ b/hadoop-hdds/docs/content/start/ProductionDeployment.zh.md
@@ -0,0 +1,69 @@
+---
+title: 生产环境部署
+weight: 6
+menu:
+ main:
+ parent: 快速入门
+---
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+本文档旨在为 Apache Ozone 的生产环境部署提供需求和最佳实践的指导。
+
+## 需求
+
+### 系统需求
+
+* **操作系统**: Linux（推荐发行版：Red Hat 8/Rocky 8+、Ubuntu、SUSE；支持架构：x86/ARM）。
+* **Java 开发工具包 (JDK)**: 版本 8 或更高。
+* **时间同步**: 必须启用时间同步服务（如 Chrony 或 ntpd）以防止时间漂移。
+
+### 存储需求
+
+* **元数据存储**: 为确保最佳性能，请使用 SAS SSD 或 NVMe SSD 存储元数据（RocksDB 和 Ratis）。
+* **DataNode 存储**: DataNode 数据存储可使用硬盘。
+* **存储类型**: 请使用直接附加存储。不要使用网络附加存储 (NAS) 或存储区域网络 (SAN)。
+
+### 网络需求
+
+* **网络带宽**: 建议网卡带宽至少为 25Gbps。
+* **网络拓扑**: 为实现可预测的性能，建议采用超分比例低于 3:1 的叶脊网络拓扑。
+
+### 安全需求 (可选但推荐)
+
+* **Kerberos**: 为增强安全性，建议使用包括密钥分发中心 (KDC) 在内的 Kerberos 环境。
+
+## 推荐配置
+
+### Linux 内核
+
+* **CPU 调节器**: 将 CPU 调节驱动设置为 `performance` 模式以最大化性能。
+* **透明大页**: 禁用透明大页以避免性能问题。
+* **SELinux**: 禁用 SELinux。
+* **Swappiness**: 设置 `vm.swappiness=1` 以最小化交换。
+
+### 本地文件系统
+
+* **LVM**: 禁用数据驱动器的逻辑卷管理器 (LVM)。
+* **文件系统**: 使用 `ext4` 或 `xfs` 文件系统。
+* **挂载选项**: 使用 `noatime` 选项挂载驱动器以减少不必要的磁盘写入。对于 SSD，还需添加 `discard` 选项。
+
+### Ozone 配置
+
+* **监控**: 安装 Prometheus 和 Grafana 以监控 Ozone 集群。
+* **管道限制**: 通过调整 `ozone.scm.datanode.pipeline.limit` 和 `ozone.scm.ec.pipeline.minimum` 来增加允许的写入管道数量，以更好地适应您的工作负载。
+* **堆大小**: 为 Ozone Manager (OM)、Storage Container Manager (SCM)、Recon、DataNode、S3 Gateway (S3G) 和 HttpFs 服务配置足够的堆大小，以确保稳定性。
diff --git a/hadoop-hdds/docs/content/tools/LogsInOzone.md b/hadoop-hdds/docs/content/tools/LogsInOzone.md
index 5196eb6a244..552d45d821f 100644
--- a/hadoop-hdds/docs/content/tools/LogsInOzone.md
+++ b/hadoop-hdds/docs/content/tools/LogsInOzone.md
@@ -1,7 +1,7 @@
---
title: "Logs in Ozone"
date: 2023-01-30
-summary: Logs in Ozone.
+summary: An overview of logging in Apache Ozone.
---
<!---
Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,16 +20,44 @@ summary: Logs in Ozone.
limitations under the License.
-->
-# AuditLog
+Apache Ozone produces different types of logs to help users monitor and troubleshoot the cluster. This document provides an overview of the available logs, their configuration, and how to use them for debugging.
-AuditLogs configurations are set in "*-audit-log4j2.properties" files. We
-can change the corresponding files to update the audit log policies for
-each component.
+## Service Logs
-## Deletion of AuditLog
+Each Ozone service (Ozone Manager, Storage Container Manager, Datanode, S3 Gateway, and Recon) generates its own log file. These logs contain detailed information about the service's operations, including errors and warnings.
-The default log appender is Rolling appender, the following configurations
-can be added for deletion of out-of-date AuditLogs.
+By default, log files are stored in the `$OZONE_LOG_DIR` directory, which is usually set to the `logs` directory under the Ozone installation. The log file names are specific to each service, for example:
+
+* `ozone-om-....log` for Ozone Manager
+* `ozone-scm-....log` for Storage Container Manager
+* `ozone-datanode-....log` for Datanode
+
+The logging behavior for each service is controlled by its `log4j.properties` file, located in the service's `$OZONE_CONF_DIR` directory, usually `etc/hadoop`. You can modify this file to change the log level, appenders, and other logging parameters.
+
+## Audit Logs
+
+Audit logs record security-sensitive operations, providing a trail of actions performed on the cluster. The following services produce audit logs:
+
+* Ozone Manager
+* Storage Container Manager
+* Datanode
+* S3 Gateway
+
+Audit log configurations are set in `*-audit-log4j2.properties` files. You can change the corresponding files to update the audit log policies for each component.
+
+### Sample Audit Log Entry
+
+Here is an example of an audit log entry from the Ozone Manager:
+
+```
+INFO | OMAudit | ? | user=hdfs | ip=127.0.0.1 | op=CREATE_VOLUME | params={volume=vol1, admin=hdfs, owner=hdfs} | result=SUCCESS
+```
+
+This entry shows that the user `hdfs` successfully created a volume named `vol1`.
+
+### Deletion of Audit Logs
+
+The default log appender is a rolling appender. The following configurations can be added for the deletion of out-of-date AuditLogs.
```
appender.rolling.strategy.type=DefaultRolloverStrategy
@@ -42,4 +70,39 @@ appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
appender.rolling.strategy.delete.ifLastModified.age=30d
```
-For more details, please check [Log4j2 Delete on Rollover](https://logging.apache.org/log4j/2.x/manual/appenders.html#CustomDeleteOnRollover).
\ No newline at end of file
+For more details, please check [Log4j2 Delete on Rollover](https://logging.apache.org/log4j/2.x/manual/appenders.html#CustomDeleteOnRollover).
+
+## Debugging
+
+You can increase the log verbosity for debugging purposes for both services and CLI tools.
+
+### Enabling Debug Logs for Services
+
+To enable debug logging for a service, you need to modify its `log4j.properties` file. Change the log level for the desired logger from `INFO` to `DEBUG`. For example, to enable debug logging for the Ozone Manager, you would edit its `log4j.properties` and change the following line:
+
+```
+rootLogger.level = info
+```
+
+to:
+
+```
+rootLogger.level = debug
+```
+
+After saving the file and restarting the service, the service will start logging more detailed debug information.
+
+### Enabling Debug Logs for CLI Tools
+
+To enable debug logging for Ozone CLI tools (e.g., `ozone sh volume create`), you can set the `OZONE_ROOT_LOGGER` environment variable to `debug`:
+
+```bash
+export OZONE_ROOT_LOGGER=DEBUG,console
+ozone sh volume create /vol1
+```
+
+Alternatively, you can use the `--loglevel` option with the `ozone` command:
+
+```bash
+ozone --loglevel debug sh volume create /vol1
+```
diff --git a/hadoop-hdds/docs/content/tools/Repair.md b/hadoop-hdds/docs/content/tools/Repair.md
new file mode 100644
index 00000000000..002b163773c
--- /dev/null
+++ b/hadoop-hdds/docs/content/tools/Repair.md
@@ -0,0 +1,252 @@
+---
+title: "Ozone Repair"
+date: 2025-07-22
+summary: Advanced tool to repair Ozone.
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Ozone Repair (`ozone repair`) is an advanced tool to repair Ozone. The nodes being repaired must be stopped before the tool is run.
+Note: All repair commands support a `--dry-run` option which allows a user to see what repair the command will be performing without actually making any changes to the cluster.
+Use the `--force` flag to override the running service check in false-positive cases.
+
+```bash
+Usage: ozone repair [-hV] [--verbose] [-conf=<configurationPath>]
+ [-D=<String=String>]... [COMMAND]
+Advanced tool to repair Ozone. The nodes being repaired must be stopped before
+the tool is run.
+ -conf=<configurationPath>
+
+ -D, --set=<String=String>
+
+ -h, --help Show this help message and exit.
+ -V, --version Print version information and exit.
+ --verbose More verbose output. Show the stack trace of the errors.
+Commands:
+ datanode Tools to repair Datanode
+ ldb Operational tool to repair ldb.
+ om Operational tool to repair OM.
+ scm Operational tool to repair SCM.
+```
+For more detailed usage see the output of `--help` for each of the subcommands.
+
+## ozone repair datanode
+Operational tool to repair datanode.
+
+### upgrade-container-schema
+Upgrade all schema V2 containers to schema V3 for a datanode in offline mode.
+Optionally takes `--volume` option to specify which volume needs the upgrade.
+
+## ozone repair ldb
+Operational tool to repair ldb.
+
+### compact
+Compact a column family in the DB to clean up tombstones while the service is offline.
+```bash
+Usage: ozone repair ldb compact [-hV] [--dry-run] [--force] [--verbose]
+ --cf=<columnFamilyName> --db=<dbPath>
+CLI to compact a column-family in the DB while the service is offline.
+Note: If om.db is compacted with this tool then it will negatively impact the
+Ozone Manager\'s efficient snapshot diff.
+ --cf, --column-family, --column_family=<columnFamilyName>
+ Column family name
+ --db=<dbPath> Database File Path
+```
+
+## ozone repair om
+Operational tool to repair OM.
+
+#### Subcommands under OM
+- fso-tree
+- snapshot
+- update-transaction
+- quota
+- compact
+- skip-ratis-transaction
+
+### fso-tree
+Identify and repair a disconnected FSO tree by marking unreferenced entries for deletion.
+Reports the reachable, unreachable (pending delete) and unreferenced (orphaned) directories and files.
+OM should be stopped while this tool is run.
+```bash
+Usage: ozone repair om fso-tree [-hV] [--dry-run] [--force] [--verbose]
+ [-b=<bucketFilter>] --db=<omDBPath>
+ [-v=<volumeFilter>]
+Identify and repair a disconnected FSO tree by marking unreferenced entries for
+deletion. OM should be stopped while this tool is run.
+ -b, --bucket=<bucketFilter>
+ Filter by bucket name
+ --db=<omDBPath> Path to OM RocksDB
+ -v, --volume=<volumeFilter>
+ Filter by volume name. Add '/' before the volume name.
+```
+
+### snapshot
+Subcommand for all snapshot related repairs.
+
+#### chain
+Update global and path previous snapshot for a snapshot in case snapshot chain is corrupted.
+```bash
+Usage: ozone repair om snapshot chain [-hV] [--dry-run] [--force] [--verbose]
+ --db=<dbPath>
+ --gp=<globalPreviousSnapshotId>
+ --pp=<pathPreviousSnapshotId> <value>
+ <snapshotName>
+CLI to update global and path previous snapshot for a snapshot in case snapshot
+chain is corrupted.
+ <value> URI of the bucket (format: volume/bucket).
+ <snapshotName> Snapshot name to update
+ --db=<dbPath> Database File Path
+ --gp, --global-previous=<globalPreviousSnapshotId>
+ Global previous snapshotId to set for the given snapshot
+ --pp, --path-previous=<pathPreviousSnapshotId>
+ Path previous snapshotId to set for the given snapshot
+```
+
+### update-transaction
+To avoid modifying Ratis logs and only update the latest applied transaction, use `update-transaction` command.
+This updates the highest transaction index in the OM transaction info table.
+```bash
+Usage: ozone repair om update-transaction [-hV] [--dry-run] [--force]
+ [--verbose] --db=<dbPath> --index=<highestTransactionIndex>
+ --term=<highestTransactionTerm>
+CLI to update the highest index in transaction info table.
+ --db=<dbPath> Database File Path
+ --index=<highestTransactionIndex>
+ Highest index to set. The input should be non-zero long
+ integer.
+ --term=<highestTransactionTerm>
+ Highest term to set. The input should be non-zero long
+ integer.
+```
+
+### quota
+Operational tool to repair quota in OM DB.
+
+#### start
+To trigger quota repair use the `start` command.
+```bash
+Usage: ozone repair om quota start [-hV] [--dry-run] [--force] [--verbose]
+ [--buckets=<buckets>]
+ [--service-host=<omHost>]
+ [--service-id=<omServiceId>]
+CLI to trigger quota repair.
+ --buckets=<buckets> start quota repair for specific buckets. Input will
+ be list of uri separated by comma as
+ /<volume>/<bucket>[,...]
+ --service-host=<omHost>
+ Ozone Manager Host. If OM HA is enabled, use
+ --service-id instead. If you must use
+ --service-host with OM HA, this must point
+ directly to the leader OM. This option is
+ required when --service-id is not provided or
+ when HA is not enabled.
+ --service-id, --om-service-id=<omServiceId>
+ Ozone Manager Service ID
+```
+
+#### status
+Get the status of last triggered quota repair.
+```bash
+Usage: ozone repair om quota status [-hV] [--verbose] [--service-host=<omHost>]
+ [--service-id=<omServiceId>]
+CLI to get the status of last trigger quota repair if available.
+ --service-host=<omHost>
+ Ozone Manager Host. If OM HA is enabled, use --service-id
+ instead. If you must use --service-host with OM HA, this
+ must point directly to the leader OM. This option is
+ required when --service-id is not provided or when HA is
+ not enabled.
+ --service-id, --om-service-id=<omServiceId>
+ Ozone Manager Service ID
+```
+
+### compact
+Compact a column family in the OM DB to clean up tombstones. The compaction happens asynchronously. Requires admin privileges.
+```bash
+Usage: ozone repair om compact [-hV] [--dry-run] [--force] [--verbose]
+ --cf=<columnFamilyName> [--node-id=<nodeId>]
+ [--service-id=<omServiceId>]
+CLI to compact a column family in the om.db. The compaction happens
+asynchronously. Requires admin privileges.
+ --cf, --column-family, --column_family=<columnFamilyName>
+ Column family name
+ --node-id=<nodeId> NodeID of the OM for which db needs to be compacted.
+ --service-id, --om-service-id=<omServiceId>
+ Ozone Manager Service ID
+```
+
+### skip-ratis-transaction, srt
+Omit a raft log in a ratis segment file by replacing the specified index with a dummy EchoOM command.
+This is an offline tool meant to be used only when all 3 OMs crash on the same transaction.
+If the issue is isolated to one OM, manually copy the DB from a healthy OM instead.
+```bash
+Usage: ozone repair om skip-ratis-transaction [-hV] [--dry-run] [--force]
+ [--verbose] -b=<backupDir> --index=<index> (-s=<segmentFile> |
+ -d=<logDir>)
+CLI to omit a raft log in a ratis segment file. The raft log at the index
+specified is replaced with an EchoOM command (which is a dummy command). It is
+an offline command i.e., doesn\'t require OM to be running. The command should
+be run for the same transaction on all 3 OMs only when all the OMs are crashing
+while applying the same transaction. If only one OM is crashing and the other
+OMs have executed the log successfully, then the DB should be manually copied
+from one of the good OMs to the crashing OM instead.
+ -b, --backup=<backupDir> Directory to put the backup of the original
+ repaired segment file before the repair.
+ -d, --ratis-log-dir=<logDir>
+ Path of the ratis log directory
+ --index=<index> Index of the failing transaction that should be
+ removed
+ -s, --segment-path=<segmentFile>
+ Path of the input segment file
+```
+
+## ozone repair scm
+Operational tool to repair SCM.
+
+#### Subcommands under SCM
+- cert
+- update-transaction
+
+### cert
+Subcommand for all certificate related repairs on SCM
+
+#### recover
+Recover Deleted SCM Certificate from RocksDB
+```bash
+Usage: ozone repair scm cert recover [-hV] [--dry-run] [--force] [--verbose]
+ --db=<dbPath>
+Recover Deleted SCM Certificate from RocksDB
+ --db=<dbPath> SCM DB Path
+```
+
+### update-transaction
+To avoid modifying Ratis logs and only update the latest applied transaction, use `update-transaction` command.
+This updates the highest transaction index in the SCM transaction info table.
+```bash
+Usage: ozone repair scm update-transaction [-hV] [--dry-run] [--force]
+ [--verbose] --db=<dbPath> --index=<highestTransactionIndex>
+ --term=<highestTransactionTerm>
+CLI to update the highest index in transaction info table.
+ --db=<dbPath> Database File Path
+ --index=<highestTransactionIndex>
+ Highest index to set. The input should be non-zero long
+ integer.
+ --term=<highestTransactionTerm>
+ Highest term to set. The input should be non-zero long
+ integer.
+```
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
index 4b9f628223f..2b891150d53 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
@@ -69,11 +69,11 @@
import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.hdds.utils.LogLevel;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.jmx.JMXJsonServlet;
-import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
index 118a17fbb5d..26c93cd1792 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
@@ -20,7 +20,6 @@
import static org.apache.hadoop.hdds.utils.HddsServerUtil.writeDBCheckpointToStream;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
-import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
import com.google.common.annotations.VisibleForTesting;
import java.io.File;
@@ -282,7 +281,7 @@ protected static Set<String> extractSstFilesToExclude(String[] sstParam) {
Set<String> receivedSstFiles = new HashSet<>();
if (sstParam != null) {
receivedSstFiles.addAll(
- Arrays.stream(sstParam).filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)).distinct().collect(Collectors.toList()));
+ Arrays.stream(sstParam).distinct().collect(Collectors.toList()));
logSstFileList(receivedSstFiles, "Received list of {} SST files to be excluded{}: {}", 5);
}
return receivedSstFiles;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
index 9c3b4fefe09..81df3a4cefb 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
@@ -23,7 +23,6 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_INFO_WAIT_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER;
-import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
import com.google.common.annotations.VisibleForTesting;
@@ -65,6 +64,7 @@
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.util.ExitUtils;
import org.apache.ratis.util.FileUtils;
@@ -318,28 +318,24 @@ public static File getMetaDir(DBDefinition definition,
}
/**
- * Scan the DB dir and return the existing SST files,
- * including omSnapshot sst files.
- * SSTs could be used for avoiding repeated download.
+ * Scan the DB dir and return the existing files,
+ * including omSnapshot files.
*
* @param db the file representing the DB to be scanned
- * @return the list of SST file name. If db not exist, will return empty list
+ * @return the list of file names. If db not exist, will return empty list
*/
- public static List<String> getExistingSstFiles(File db) throws IOException {
+ public static List<String> getExistingFiles(File db) throws IOException {
List<String> sstList = new ArrayList<>();
if (!db.exists()) {
return sstList;
}
-
- int truncateLength = db.toString().length() + 1;
// Walk the db dir and get all sst files including omSnapshot files.
try (Stream<Path> files = Files.walk(db.toPath())) {
- sstList =
- files.filter(path -> path.toString().endsWith(ROCKSDB_SST_SUFFIX)).
- map(p -> p.toString().substring(truncateLength)).
+ sstList = files.filter(p -> p.toFile().isFile())
+ .map(p -> p.getFileName().toString()).
collect(Collectors.toList());
if (LOG.isDebugEnabled()) {
- LOG.debug("Scanned SST files {} in {}.", sstList, db.getAbsolutePath());
+ LOG.debug("Scanned files {} in {}.", sstList, db.getAbsolutePath());
}
}
return sstList;
@@ -347,21 +343,45 @@ public static List<String> getExistingSstFiles(File db) throws IOException {
/**
* Retry forever until CA list matches expected count.
+ * Fails fast on authentication exceptions.
* @param task - task to get CA list.
* @return CA list.
*/
private static List<String> getCAListWithRetry(Callable<List<String>> task,
long waitDuration) throws IOException {
- RetryPolicy retryPolicy = RetryPolicies.retryForeverWithFixedSleep(
- waitDuration, TimeUnit.SECONDS);
- RetriableTask<List<String>> retriableTask =
- new RetriableTask<>(retryPolicy, "getCAList", task);
+ RetryPolicy retryPolicy = new RetryPolicy() {
+ private final RetryPolicy defaultPolicy = RetryPolicies.retryForeverWithFixedSleep(
+ waitDuration, TimeUnit.SECONDS);
+
+ @Override
+ public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotent) throws Exception {
+ if (containsAccessControlException(e)) {
+ LOG.warn("AccessControlException encountered during getCAList; failing fast without retry.");
+ return new RetryAction(RetryAction.RetryDecision.FAIL);
+ }
+ return defaultPolicy.shouldRetry(e, retries, failovers, isIdempotent);
+ }
+ };
+
+ RetriableTask<List<String>> retriableTask = new RetriableTask<>(retryPolicy, "getCAList", task);
try {
return retriableTask.call();
} catch (Exception ex) {
- throw new SCMSecurityException("Unable to obtain complete CA " +
- "list", ex);
+ if (containsAccessControlException(ex)) {
+ throw new AccessControlException();
+ }
+ throw new SCMSecurityException("Unable to obtain complete CA list", ex);
+ }
+ }
+
+ private static boolean containsAccessControlException(Throwable e) {
+ while (e != null) {
+ if (e instanceof AccessControlException) {
+ return true;
+ }
+ e = e.getCause();
}
+ return false;
}
private static List<String> waitForCACerts(
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LogLevel.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LogLevel.java
new file mode 100644
index 00000000000..0ed1682dccc
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LogLevel.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.server.http.HttpServer2;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Change log level in runtime.
+ */
+@InterfaceStability.Evolving
+public final class LogLevel {
+ private static final String USAGES = "\nUsage: Command options are:\n"
+ + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
+ + "\t[-setlevel <host:port> <classname> <level> "
+ + "[-protocol (http|https)]\n";
+
+ private static final String PROTOCOL_HTTP = "http";
+ private static final String PROTOCOL_HTTPS = "https";
+
+ private static final String SLF4J_LOG4J_ADAPTER_CLASS = "org.slf4j.reload4j.Reload4jLoggerAdapter";
+
+ private static final String MARKER = "<!-- OUTPUT -->";
+ private static final Pattern TAG = Pattern.compile("<[^>]*>");
+
+ /**
+ * Set to false only if log4j adapter class is not found in the classpath. Once set to false,
+ * the utility method should not bother re-loading class again.
+ */
+ private static final AtomicBoolean IS_LOG4J_LOGGER = new AtomicBoolean(true);
+
+ /**
+ * A command line implementation.
+ * @param args input args.
+ * @throws Exception exception.
+ */
+ public static void main(String[] args) throws Exception {
+ CLI cli = new CLI(new Configuration());
+ System.exit(ToolRunner.run(cli, args));
+ }
+
+ private LogLevel() {
+ // no instances
+ }
+
+ /**
+ * Valid command line options.
+ */
+ private enum Operations {
+ GETLEVEL,
+ SETLEVEL,
+ UNKNOWN
+ }
+
+ private static void printUsage() {
+ System.err.println(USAGES);
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ }
+
+ public static boolean isValidProtocol(String protocol) {
+ return ((protocol.equals(PROTOCOL_HTTP) ||
+ protocol.equals(PROTOCOL_HTTPS)));
+ }
+
+ /**
+ * Determine whether the log of the given logger is of Log4J implementation.
+ *
+ * @param logger the logger name, usually class name as string.
+ * @return true if the logger uses Log4J implementation.
+ */
+ private static boolean isLog4jLogger(String logger) {
+ if (logger == null || !IS_LOG4J_LOGGER.get()) {
+ return false;
+ }
+ org.slf4j.Logger log = LoggerFactory.getLogger(logger);
+ try {
+ Class<?> log4jClass = Class.forName(SLF4J_LOG4J_ADAPTER_CLASS);
+ return log4jClass.isInstance(log);
+ } catch (ClassNotFoundException e) {
+ IS_LOG4J_LOGGER.set(false);
+ return false;
+ }
+ }
+
+ @VisibleForTesting
+ static class CLI extends Configured implements Tool {
+ private Operations operation = Operations.UNKNOWN;
+ private String protocol;
+ private String hostName;
+ private String className;
+ private String level;
+
+ CLI(Configuration conf) {
+ setConf(conf);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ try {
+ parseArguments(args);
+ sendLogLevelRequest();
+ } catch (HadoopIllegalArgumentException e) {
+ printUsage();
+ return -1;
+ }
+ return 0;
+ }
+
+ /**
+ * Send HTTP/HTTPS request to the daemon.
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void sendLogLevelRequest()
+ throws HadoopIllegalArgumentException, Exception {
+ switch (operation) {
+ case GETLEVEL:
+ doGetLevel();
+ break;
+ case SETLEVEL:
+ doSetLevel();
+ break;
+ default:
+ throw new HadoopIllegalArgumentException(
+ "Expect either -getlevel or -setlevel");
+ }
+ }
+
+ public void parseArguments(String[] args) throws
+ HadoopIllegalArgumentException {
+ if (args.length == 0) {
+ throw new HadoopIllegalArgumentException("No arguments specified");
+ }
+ int nextArgIndex = 0;
+ while (nextArgIndex < args.length) {
+ if (args[nextArgIndex].equals("-getlevel")) {
+ nextArgIndex = parseGetLevelArgs(args, nextArgIndex);
+ } else if (args[nextArgIndex].equals("-setlevel")) {
+ nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
+ } else if (args[nextArgIndex].equals("-protocol")) {
+ nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+ } else {
+ throw new HadoopIllegalArgumentException(
+ "Unexpected argument " + args[nextArgIndex]);
+ }
+ }
+
+ // if operation is never specified in the arguments
+ if (operation == Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Must specify either -getlevel or -setlevel");
+ }
+
+ // if protocol is unspecified, set it as http.
+ if (protocol == null) {
+ protocol = PROTOCOL_HTTP;
+ }
+ }
+
+ private int parseGetLevelArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // fail if multiple operations are specified in the arguments
+ if (operation != Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -getlevel command");
+ }
+ // check number of arguments is sufficient
+ if (index + 2 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-getlevel needs two parameters");
+ }
+ operation = Operations.GETLEVEL;
+ hostName = args[index + 1];
+ className = args[index + 2];
+ return index + 3;
+ }
+
+ private int parseSetLevelArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // fail if multiple operations are specified in the arguments
+ if (operation != Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -setlevel command");
+ }
+ // check number of arguments is sufficient
+ if (index + 3 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-setlevel needs three parameters");
+ }
+ operation = Operations.SETLEVEL;
+ hostName = args[index + 1];
+ className = args[index + 2];
+ level = args[index + 3];
+ return index + 4;
+ }
+
+ private int parseProtocolArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // make sure only -protocol is specified
+ if (protocol != null) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -protocol command");
+ }
+ // check number of arguments is sufficient
+ if (index + 1 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-protocol needs one parameter");
+ }
+ // check protocol is valid
+ protocol = args[index + 1];
+ if (!isValidProtocol(protocol)) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid protocol: " + protocol);
+ }
+ return index + 2;
+ }
+
+ /**
+ * Send HTTP/HTTPS request to get log level.
+ *
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void doGetLevel() throws Exception {
+ process(protocol + "://" + hostName + "/logLevel?log=" + className);
+ }
+
+ /**
+ * Send HTTP/HTTPS request to set log level.
+ *
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void doSetLevel() throws Exception {
+ process(protocol + "://" + hostName + "/logLevel?log=" + className
+ + "&level=" + level);
+ }
+
+ /**
+ * Connect to the URL. Supports HTTP/HTTPS and supports SPNEGO
+ * authentication. It falls back to simple authentication if it fails to
+ * initiate SPNEGO.
+ *
+ * @param url the URL address of the daemon servlet
+ * @return a connected connection
+ * @throws Exception if it can not establish a connection.
+ */
+ private URLConnection connect(URL url) throws Exception {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ AuthenticatedURL aUrl;
+ SSLFactory clientSslFactory;
+ URLConnection connection;
+ // If https is chosen, configures SSL client.
+ if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+ clientSslFactory = new SSLFactory(
+ SSLFactory.Mode.CLIENT, this.getConf());
+ clientSslFactory.init();
+ SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+
+ aUrl = new AuthenticatedURL(
+ new KerberosAuthenticator(), clientSslFactory);
+ connection = aUrl.openConnection(url, token);
+ HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+ httpsConn.setSSLSocketFactory(sslSocketF);
+ } else {
+ aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+ connection = aUrl.openConnection(url, token);
+ }
+
+ connection.connect();
+ return connection;
+ }
+
+ /**
+ * Configures the client to send HTTP/HTTPS request to the URL.
+ * Supports SPENGO for authentication.
+ * @param urlString URL and query string to the daemon's web UI
+ * @throws Exception if unable to connect
+ */
+ private void process(String urlString) throws Exception {
+ URL url = new URL(urlString);
+ System.out.println("Connecting to " + url);
+
+ URLConnection connection = connect(url);
+
+ // read from the servlet
+ BufferedReader in = new BufferedReader(
+ new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8));
+ for (String line;;) {
+ line = in.readLine();
+ if (line == null) {
+ break;
+ }
+ if (line.startsWith(MARKER)) {
+ System.out.println(TAG.matcher(line).replaceAll(""));
+ }
+ }
+ in.close();
+ }
+ }
+
+ /**
+ * A servlet implementation.
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+ @InterfaceStability.Unstable
+ public static class Servlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+
+ private static final String FORMS = "\n<br /><hr /><h3>Get / Set</h3>"
+ + "\n<form>Class Name: <input type='text' size='50' name='log' /> "
+ + "<input type='submit' value='Get Log Level' />"
+ + "</form>"
+ + "\n<form>Class Name: <input type='text' size='50' name='log' /> "
+ + "Level: <input type='text' name='level' /> "
+ + "<input type='submit' value='Set Log Level' />"
+ + "</form>";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+
+ // Do the authorization
+ if (!HttpServer2.hasAdministratorAccess(getServletContext(), request,
+ response)) {
+ return;
+ }
+
+ PrintWriter out = ServletUtil.initHTML(response, "Log Level");
+ String logName = ServletUtil.getParameter(request, "log");
+ String level = ServletUtil.getParameter(request, "level");
+
+ if (logName != null) {
+ out.println("<br /><hr /><h3>Results</h3>");
+ out.println(MARKER
+ + "Submitted Class Name: <b>" + logName + "</b><br />");
+
+ org.slf4j.Logger log = LoggerFactory.getLogger(logName);
+ out.println(MARKER
+ + "Log Class: <b>" + log.getClass().getName() + "</b><br />");
+ if (level != null) {
+ out.println(MARKER + "Submitted Level: <b>" + level + "</b><br />");
+ }
+
+ if (isLog4jLogger(logName)) {
+ process(Logger.getLogger(logName), level, out);
+ } else {
+ out.println("Sorry, setting log level is only supported for log4j loggers.<br />");
+ }
+ }
+
+ out.println(FORMS);
+ out.println(ServletUtil.HTML_TAIL);
+ }
+
+ private static void process(Logger log, String level,
+ PrintWriter out) throws IOException {
+ if (level != null) {
+ if (!level.equalsIgnoreCase(Level.toLevel(level)
+ .toString())) {
+ out.println(MARKER + "Bad Level : <b>" + level + "</b><br />");
+ } else {
+ log.setLevel(Level.toLevel(level));
+ out.println(MARKER + "Setting Level to " + level + " ...<br />");
+ }
+ }
+ out.println(MARKER
+ + "Effective Level: <b>" + log.getEffectiveLevel() + "</b><br />");
+ }
+
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
index 777efcf47ea..b40a1f84e15 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
@@ -108,14 +108,15 @@ public DBCheckpoint downloadDBSnapshotFromLeader(String leaderNodeID)
LOG.info("Prepare to download the snapshot from leader OM {} and " +
"reloading state from the snapshot.", leaderNodeID);
checkLeaderConsistency(leaderNodeID);
+ int numParts = 0;
while (true) {
String snapshotFileName = getSnapshotFileName(leaderNodeID);
File targetFile = new File(snapshotDir, snapshotFileName);
downloadSnapshot(leaderNodeID, targetFile);
- LOG.info(
- "Successfully download the latest snapshot {} from leader OM: {}",
- targetFile, leaderNodeID);
+ LOG.info("Successfully download the latest snapshot {} from leader OM: {}, part : {}",
+ targetFile, leaderNodeID, numParts);
+ numParts++;
numDownloaded.incrementAndGet();
injectPause();
@@ -153,7 +154,7 @@ void checkLeaderConsistency(String currentLeader) throws IOException {
return;
}
- List<String> files = HAUtils.getExistingSstFiles(candidateDir);
+ List<String> files = HAUtils.getExistingFiles(candidateDir);
if (!files.isEmpty()) {
LOG.warn("Candidate DB directory {} is not empty when last leader is " +
"null.", candidateDir);
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
index 5af0e671d51..d3fbb3cb9f2 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.hdds.utils;
+import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
@@ -41,7 +42,12 @@ public MapBackedTableIterator(TreeMap<String, V> values, String prefix) {
public void seekToFirst() {
this.itr = this.values.entrySet().stream()
.filter(e -> prefix == null || e.getKey().startsWith(prefix))
- .map(e -> Table.newKeyValue(e.getKey(), e.getValue())).iterator();
+ .map(e -> {
+ V value = e.getValue();
+ int size = value != null ? value.toString().getBytes(StandardCharsets.UTF_8).length : 0;
+ return Table.newKeyValue(e.getKey(), value, size);
+ })
+ .iterator();
}
@Override
@@ -53,8 +59,8 @@ public void seekToLast() {
public Table.KeyValue<String, V> seek(String s) {
this.itr = this.values.entrySet().stream()
.filter(e -> prefix == null || e.getKey().startsWith(prefix))
- .filter(e -> e.getKey().compareTo(s) >= 0)
- .map(e -> Table.newKeyValue(e.getKey(), e.getValue())).iterator();
+ .filter(e -> e.getKey().compareTo(s) >= 0).map(e -> Table.newKeyValue(e.getKey(), e.getValue(),
+ e.getValue().toString().getBytes(StandardCharsets.UTF_8).length)).iterator();
Map.Entry<String, V> firstEntry = values.ceilingEntry(s);
return firstEntry == null ? null : Table.newKeyValue(firstEntry.getKey(), firstEntry.getValue());
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java
index 652c1443b7b..458a87b6682 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java
@@ -116,9 +116,11 @@ public void downloadSnapshot(String leaderNodeID, File targetFile)
concat(String.valueOf(a.length())))
.collect(Collectors.toList()));
try (OutputStream outputStream = Files.newOutputStream(targetFile.toPath())) {
- writeDBCheckpointToStream(dbCheckpoint, outputStream,
- new HashSet<>(HAUtils.getExistingSstFiles(
- rdbSnapshotProvider.getCandidateDir())));
+ Set<String> existingSstFiles = HAUtils.getExistingFiles(rdbSnapshotProvider.getCandidateDir())
+ .stream()
+ .filter(fName -> fName.endsWith(".sst") && !fName.equals(".sst"))
+ .collect(Collectors.toSet());
+ writeDBCheckpointToStream(dbCheckpoint, outputStream, existingSstFiles);
}
}
};
@@ -141,7 +143,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception {
assertTrue(candidateDir.exists());
DBCheckpoint checkpoint;
- int before = HAUtils.getExistingSstFiles(
+ int before = HAUtils.getExistingFiles(
rdbSnapshotProvider.getCandidateDir()).size();
assertEquals(0, before);
@@ -149,12 +151,12 @@ public void testDownloadDBSnapshotFromLeader() throws Exception {
checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID);
File checkpointDir = checkpoint.getCheckpointLocation().toFile();
assertEquals(candidateDir, checkpointDir);
- int first = HAUtils.getExistingSstFiles(
+ int first = HAUtils.getExistingFiles(
rdbSnapshotProvider.getCandidateDir()).size();
// Get second snapshot
checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID);
- int second = HAUtils.getExistingSstFiles(
+ int second = HAUtils.getExistingFiles(
rdbSnapshotProvider.getCandidateDir()).size();
assertThat(second).withFailMessage("The second snapshot should have more SST files")
.isGreaterThan(first);
@@ -164,7 +166,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception {
// Get third snapshot
checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID);
- int third = HAUtils.getExistingSstFiles(
+ int third = HAUtils.getExistingFiles(
rdbSnapshotProvider.getCandidateDir()).size();
assertThat(third).withFailMessage("The third snapshot should have more SST files")
.isGreaterThan(second);
@@ -173,7 +175,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception {
// Test cleanup candidateDB
rdbSnapshotProvider.init();
- assertEquals(0, HAUtils.getExistingSstFiles(
+ assertEquals(0, HAUtils.getExistingFiles(
rdbSnapshotProvider.getCandidateDir()).size());
}
diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
index 1ee5ece03d2..ea567948b91 100644
--- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
@@ -592,3 +592,7 @@ service IntraDatanodeProtocolService {
rpc download (CopyContainerRequestProto) returns (stream CopyContainerResponseProto);
rpc upload (stream SendContainerRequest) returns (SendContainerResponse);
}
+
+message ContainerCreateInfo {
+ required ContainerDataProto.State state = 1;
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedConfigOptions.java
similarity index 62%
copy from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java
copy to hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedConfigOptions.java
index 4d58158fb2a..aef0cddc31e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java
+++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedConfigOptions.java
@@ -15,16 +15,26 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.om.snapshot;
+package org.apache.hadoop.hdds.utils.db.managed;
-import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
+import static org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils.track;
+
+import org.apache.ratis.util.UncheckedAutoCloseable;
+import org.rocksdb.ConfigOptions;
/**
- * Test OmSnapshot for FSO bucket type when native lib is disabled.
+ * Managed ConfigOptions.
*/
-public class TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets extends TestOmSnapshot {
+public class ManagedConfigOptions extends ConfigOptions {
+
+ private final UncheckedAutoCloseable leakTracker = track(this);
- public TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets() throws Exception {
- super(FILE_SYSTEM_OPTIMIZED, false, false, true, true);
+ @Override
+ public void close() {
+ try {
+ super.close();
+ } finally {
+ leakTracker.close();
+ }
}
}
diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
index 0fff6098bff..3401469f682 100644
--- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
+++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
@@ -28,6 +28,7 @@
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.DBOptions;
import org.rocksdb.LiveFileMetaData;
+import org.rocksdb.OptionsUtil;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
@@ -85,13 +86,28 @@ public static ManagedRocksDB open(
);
}
- public static ManagedRocksDB open(
- final String path,
+ /**
+ * Open a RocksDB option with the latest options. Other than {@link ManagedConfigOptions} and
+ * path, the other options and handles should be empty since it will be populated with
+ * loadLatestOptions. Nevertheless, the caller is still responsible in cleaning up / closing the resources.
+ * @param configOptions Config options.
+ * @param options DBOptions. This would be modified based on the latest options.
+ * @param path DB path.
+ * @param columnFamilyDescriptors Column family descriptors. These would be modified based on the latest options.
+ * @param columnFamilyHandles Column family handles of the underlying column family
+ * @return A RocksDB instance.
+ * @throws RocksDBException thrown if error happens in underlying native library.
+ */
+ public static ManagedRocksDB openWithLatestOptions(
+ final ManagedConfigOptions configOptions,
+ final DBOptions options, final String path,
final List<ColumnFamilyDescriptor> columnFamilyDescriptors,
final List<ColumnFamilyHandle> columnFamilyHandles)
throws RocksDBException {
+ // Preserve all the previous DB options
+ OptionsUtil.loadLatestOptions(configOptions, path, options, columnFamilyDescriptors);
return new ManagedRocksDB(
- RocksDB.open(path, columnFamilyDescriptors, columnFamilyHandles)
+ RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles)
);
}
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
index fd8aae95c13..df94ff35af8 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
@@ -84,7 +84,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>test</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
index 583a3bb7bc1..45cd4d6a79b 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
@@ -172,6 +172,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable,
= new BootstrapStateHandler.Lock();
private static final int SST_READ_AHEAD_SIZE = 2 * 1024 * 1024;
private int pruneSSTFileBatchSize;
+ private SSTFilePruningMetrics sstFilePruningMetrics;
private ColumnFamilyHandle snapshotInfoTableCFHandle;
private static final String DAG_PRUNING_SERVICE_NAME = "CompactionDagPruningService";
private AtomicBoolean suspended;
@@ -239,10 +240,11 @@ public class RocksDBCheckpointDiffer implements AutoCloseable,
this.pruneSSTFileBatchSize = configuration.getInt(
OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE,
OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_BACKUP_BATCH_SIZE_DEFAULT);
+ this.sstFilePruningMetrics = SSTFilePruningMetrics.create(activeDBLocationName);
try {
if (configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)
&& ManagedRawSSTFileReader.loadLibrary()) {
- pruneQueue = new ConcurrentLinkedQueue<>();
+ this.pruneQueue = new ConcurrentLinkedQueue<>();
}
} catch (NativeLibraryNotLoadedException e) {
LOG.warn("Native Library for raw sst file reading loading failed." +
@@ -338,6 +340,9 @@ public void close() {
LOG.info("Shutting down {}.", DAG_PRUNING_SERVICE_NAME);
scheduler.close();
}
+ if (sstFilePruningMetrics != null) {
+ sstFilePruningMetrics.unRegister();
+ }
}
}
}
@@ -532,6 +537,7 @@ public void onCompactionCompleted(RocksDB db,
// so that the backup input sst files can be pruned.
if (pruneQueue != null) {
pruneQueue.offer(key);
+ sstFilePruningMetrics.updateQueueSize(pruneQueue.size());
}
}
};
@@ -751,6 +757,10 @@ private void loadCompactionDagFromDB() {
}
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
+ } finally {
+ if (pruneQueue != null) {
+ sstFilePruningMetrics.updateQueueSize(pruneQueue.size());
+ }
}
}
@@ -1258,13 +1268,16 @@ public void pruneSstFileValues() {
if (!shouldRun()) {
return;
}
+ long batchStartTime = System.nanoTime();
+ int filesPrunedInBatch = 0;
+ int filesSkippedInBatch = 0;
+ int batchCounter = 0;
Path sstBackupDirPath = Paths.get(sstBackupDir);
Path prunedSSTFilePath = sstBackupDirPath.resolve(PRUNED_SST_FILE_TEMP);
try (ManagedOptions managedOptions = new ManagedOptions();
ManagedEnvOptions envOptions = new ManagedEnvOptions()) {
byte[] compactionLogEntryKey;
- int batchCounter = 0;
while ((compactionLogEntryKey = pruneQueue.peek()) != null && ++batchCounter <= pruneSSTFileBatchSize) {
CompactionLogEntry compactionLogEntry;
// Get the compaction log entry.
@@ -1289,6 +1302,7 @@ public void pruneSstFileValues() {
if (Files.notExists(sstFilePath)) {
LOG.debug("Skipping pruning SST file {} as it does not exist in backup directory.", sstFilePath);
updatedFileInfoList.add(fileInfo);
+ filesSkippedInBatch++;
continue;
}
@@ -1306,6 +1320,7 @@ public void pruneSstFileValues() {
fileInfo.setPruned();
updatedFileInfoList.add(fileInfo);
LOG.debug("Completed pruning OMKeyInfo from {}", sstFilePath);
+ filesPrunedInBatch++;
}
// Update compaction log entry in table.
@@ -1325,6 +1340,12 @@ public void pruneSstFileValues() {
}
} catch (IOException | InterruptedException e) {
LOG.error("Could not prune source OMKeyInfo from backup SST files.", e);
+ sstFilePruningMetrics.incrPruningFailures();
+ } finally {
+ LOG.info("Completed pruning OMKeyInfo from backup SST files in {}ms.",
+ (System.nanoTime() - batchStartTime) / 1_000_000);
+ sstFilePruningMetrics.updateBatchLevelMetrics(filesPrunedInBatch, filesSkippedInBatch,
+ batchCounter, pruneQueue.size());
}
}
@@ -1428,4 +1449,9 @@ private Map<String, CompactionFileInfo> toFileInfoList(List<String> sstFiles, Ro
ConcurrentMap<String, CompactionFileInfo> getInflightCompactions() {
return inflightCompactions;
}
+
+ @VisibleForTesting
+ public SSTFilePruningMetrics getPruningMetrics() {
+ return sstFilePruningMetrics;
+ }
}
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/SSTFilePruningMetrics.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/SSTFilePruningMetrics.java
new file mode 100644
index 00000000000..46eb2c0889b
--- /dev/null
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/SSTFilePruningMetrics.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.rocksdiff;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class contains metrics for monitoring SST file pruning operations in RocksDBCheckpointDiffer.
+ */
+@Metrics(about = "SST File Pruning Metrics", context = OzoneConsts.OZONE)
+public final class SSTFilePruningMetrics implements MetricsSource {
+
+ private static final String METRICS_SOURCE_NAME_PREFIX = SSTFilePruningMetrics.class.getSimpleName();
+ private final String metricSourceName;
+ private final MetricsRegistry registry;
+
+ /*
+ * Pruning Throughput Metrics.
+ */
+ @Metric("Total no. of SST files pruned")
+ private MutableCounterLong filesPrunedTotal;
+ @Metric("No. of SST files pruned in the last batch")
+ private MutableGaugeLong filesPrunedLast;
+ @Metric("Total no. of SST files removed")
+ private MutableCounterLong filesSkippedTotal;
+ @Metric("Total no. of compactions processed")
+ private MutableCounterLong compactionsProcessed;
+ @Metric("No. of pending pruning jobs in queue")
+ private MutableGaugeLong pruneQueueSize;
+
+ /*
+ * Pruning failure Metrics.
+ */
+ @Metric("No. of pruning job failures")
+ private MutableCounterLong pruningFailures;
+
+ private SSTFilePruningMetrics(String sourceName) {
+ this.metricSourceName = sourceName;
+ this.registry = new MetricsRegistry(metricSourceName);
+ }
+
+ /**
+ * Creates and returns SSTFilePruningMetrics instance.
+ *
+ * @return SSTFilePruningMetrics
+ */
+ public static SSTFilePruningMetrics create(String dbLocation) {
+ String sourceName = METRICS_SOURCE_NAME_PREFIX +
+ (dbLocation == null || dbLocation.isEmpty() ? "" : "-" + dbLocation.replaceAll("[/\\:\\s]", "_"));
+ return DefaultMetricsSystem.instance().register(sourceName, "SST File Pruning Metrics",
+ new SSTFilePruningMetrics(sourceName));
+ }
+
+ /**
+ * Unregister the metrics instance.
+ */
+ public void unRegister() {
+ DefaultMetricsSystem.instance().unregisterSource(metricSourceName);
+ }
+
+ public void updateQueueSize(long queueSize) {
+ pruneQueueSize.set(queueSize);
+ }
+
+ public void updateBatchLevelMetrics(long filesPruned, long filesSkipped, int compactions, long queueSize) {
+ filesPrunedTotal.incr(filesPruned);
+ filesPrunedLast.set(filesPruned);
+ filesSkippedTotal.incr(filesSkipped);
+ compactionsProcessed.incr(compactions);
+ updateQueueSize(queueSize);
+ }
+
+ public void incrPruningFailures() {
+ pruningFailures.incr();
+ }
+
+ public long getFilesPrunedTotal() {
+ return filesPrunedTotal.value();
+ }
+
+ public long getFilesPrunedLast() {
+ return filesPrunedLast.value();
+ }
+
+ public long getFilesRemovedTotal() {
+ return filesSkippedTotal.value();
+ }
+
+ public long getCompactionsProcessed() {
+ return compactionsProcessed.value();
+ }
+
+ public long getPruneQueueSize() {
+ return pruneQueueSize.value();
+ }
+
+ public long getPruningFailures() {
+ return pruningFailures.value();
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ MetricsRecordBuilder recordBuilder = collector.addRecord(metricSourceName);
+ filesPrunedTotal.snapshot(recordBuilder, all);
+ filesPrunedLast.snapshot(recordBuilder, all);
+ filesSkippedTotal.snapshot(recordBuilder, all);
+ compactionsProcessed.snapshot(recordBuilder, all);
+ pruneQueueSize.snapshot(recordBuilder, all);
+ pruningFailures.snapshot(recordBuilder, all);
+ }
+}
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
index 1a329647cc9..5b6fc39f237 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
@@ -1935,6 +1935,12 @@ private static Stream<Arguments> casesGetSSTDiffListWithoutDB2() {
*/
@Test
public void testPruneSSTFileValues() throws Exception {
+ SSTFilePruningMetrics sstFilePruningMetrics = rocksDBCheckpointDiffer.getPruningMetrics();
+ assertEquals(0L, sstFilePruningMetrics.getPruneQueueSize());
+ assertEquals(0L, sstFilePruningMetrics.getFilesPrunedTotal());
+ assertEquals(0L, sstFilePruningMetrics.getFilesPrunedLast());
+ assertEquals(0L, sstFilePruningMetrics.getCompactionsProcessed());
+ assertEquals(0L, sstFilePruningMetrics.getFilesRemovedTotal());
List<Pair<byte[], Integer>> keys = new ArrayList<Pair<byte[], Integer>>();
keys.add(Pair.of("key1".getBytes(UTF_8), Integer.valueOf(1)));
@@ -1962,8 +1968,9 @@ public void testPruneSSTFileValues() throws Exception {
);
byte[] compactionLogEntryKey = rocksDBCheckpointDiffer.addToCompactionLogTable(compactionLogEntry);
rocksDBCheckpointDiffer.loadAllCompactionLogs();
+ assertEquals(1L, sstFilePruningMetrics.getPruneQueueSize());
- // Pruning should not fail a source SST file has been removed by a another pruner.
+ // Pruning should not fail a source SST file has been removed by another pruner.
Files.delete(sstBackUpDir.toPath().resolve(inputFile73 + SST_FILE_EXTENSION));
// Run the SST file pruner.
ManagedRawSSTFileIterator mockedRawSSTFileItr = mock(ManagedRawSSTFileIterator.class);
@@ -2011,6 +2018,12 @@ public void testPruneSSTFileValues() throws Exception {
// Verify 000073.sst pruning has been skipped
assertFalse(fileInfo73.isPruned());
+
+ assertEquals(0L, sstFilePruningMetrics.getPruneQueueSize());
+ assertEquals(1L, sstFilePruningMetrics.getFilesPrunedTotal());
+ assertEquals(1L, sstFilePruningMetrics.getFilesPrunedLast());
+ assertEquals(1L, sstFilePruningMetrics.getCompactionsProcessed());
+ assertEquals(1L, sstFilePruningMetrics.getFilesRemovedTotal());
}
private void createSSTFileWithKeys(String filePath, List<Pair<byte[], Integer>> keys)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
index 989971492a8..06a003f1866 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
@@ -68,6 +68,14 @@ List<String> getTransactionIDList(DatanodeID dnId) {
.collect(Collectors.toList());
}
+ public int getNumberOfBlocksForDatanode(DatanodeID dnId) {
+ return Optional.ofNullable(transactions.get(dnId))
+ .orElse(new LinkedList<>())
+ .stream()
+ .mapToInt(DeletedBlocksTransaction::getLocalIDCount)
+ .sum();
+ }
+
boolean isEmpty() {
return transactions.isEmpty();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index 3d4509b9d87..c94036b9cbc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -19,6 +19,8 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR_DEFAULT;
import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus;
import static org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator.DEL_TXN_ID;
@@ -93,6 +95,7 @@ public class DeletedBlockLogImpl
private static final int LIST_ALL_FAILED_TRANSACTIONS = -1;
private long lastProcessedTransactionId = -1;
private final int logAppenderQueueByteLimit;
+ private int deletionFactorPerDatanode;
public DeletedBlockLogImpl(ConfigurationSource conf,
StorageContainerManager scm,
@@ -123,6 +126,10 @@ public DeletedBlockLogImpl(ConfigurationSource conf,
ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
StorageUnit.BYTES);
this.logAppenderQueueByteLimit = (int) (limit * 0.9);
+ int deletionFactor = conf.getInt(OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR,
+ OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR_DEFAULT);
+ this.deletionFactorPerDatanode = deletionFactor <= 0 ? 1 : deletionFactor;
+
}
@VisibleForTesting
@@ -130,6 +137,11 @@ void setDeletedBlockLogStateManager(DeletedBlockLogStateManager manager) {
this.deletedBlockLogStateManager = manager;
}
+ @VisibleForTesting
+ void setDeleteBlocksFactorPerDatanode(int deleteBlocksFactorPerDatanode) {
+ this.deletionFactorPerDatanode = deleteBlocksFactorPerDatanode;
+ }
+
@Override
public List<DeletedBlocksTransaction> getFailedTransactions(int count,
long startTxId) throws IOException {
@@ -328,7 +340,16 @@ public void close() {
private void getTransaction(DeletedBlocksTransaction tx,
DatanodeDeletedBlockTransactions transactions,
Set<ContainerReplica> replicas,
- Map<DatanodeID, Map<Long, CmdStatus>> commandStatus) {
+ Map<DatanodeID, Map<Long, CmdStatus>> commandStatus,
+ int maxDeleteBlocksPerDatanode) {
+ // Ensure all DNs for this transaction are below their max block limit.
+ if (!replicas.stream().allMatch(replica -> {
+ final DatanodeID datanodeID = replica.getDatanodeDetails().getID();
+ return transactions.getNumberOfBlocksForDatanode(datanodeID) < maxDeleteBlocksPerDatanode;
+ })) {
+ return;
+ }
+
DeletedBlocksTransaction updatedTxn =
DeletedBlocksTransaction.newBuilder(tx)
.setCount(transactionStatusManager.getRetryCount(tx.getTxID()))
@@ -430,6 +451,12 @@ public DatanodeDeletedBlockTransactions getTransactions(
ArrayList<Long> txIDs = new ArrayList<>();
metrics.setNumBlockDeletionTransactionDataNodes(dnList.size());
Table.KeyValue<Long, DeletedBlocksTransaction> keyValue = null;
+
+ int factor = dnList.size() / deletionFactorPerDatanode;
+ int maxDeleteBlocksPerDatanode = (factor > 0)
+ ? Math.min(blockDeletionLimit, blockDeletionLimit / factor)
+ : blockDeletionLimit;
+
// Here takes block replica count as the threshold to avoid the case
// that part of replicas committed the TXN and recorded in the
// SCMDeletedBlockTransactionStatusManager, while they are counted
@@ -451,11 +478,11 @@ public DatanodeDeletedBlockTransactions getTransactions(
Set<ContainerReplica> replicas = containerManager
.getContainerReplicas(
ContainerID.valueOf(txn.getContainerID()));
- if (checkInadequateReplica(replicas, txn, dnList)) {
+ if (!checkInadequateReplica(replicas, txn, dnList)) {
+ getTransaction(txn, transactions, replicas, commandStatus, maxDeleteBlocksPerDatanode);
+ } else {
metrics.incrSkippedTransaction();
- continue;
}
- getTransaction(txn, transactions, replicas, commandStatus);
} else if (txn.getCount() >= maxRetry || containerManager.getContainer(id).isOpen()) {
metrics.incrSkippedTransaction();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java
index 3c3e6fd712b..2be951a1814 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java
@@ -30,19 +30,29 @@ public class ContainerReplicaOp {
private final int replicaIndex;
private final SCMCommand<?> command;
private final long deadlineEpochMillis;
+ private final long containerSize;
public static ContainerReplicaOp create(PendingOpType opType,
DatanodeDetails target, int replicaIndex) {
- return new ContainerReplicaOp(opType, target, replicaIndex, null, System.currentTimeMillis());
+ return new ContainerReplicaOp(opType, target, replicaIndex, null,
+ System.currentTimeMillis(), 0);
}
public ContainerReplicaOp(PendingOpType opType,
- DatanodeDetails target, int replicaIndex, SCMCommand<?> command, long deadlineEpochMillis) {
+ DatanodeDetails target, int replicaIndex, SCMCommand<?> command,
+ long deadlineEpochMillis) {
+ this(opType, target, replicaIndex, command, deadlineEpochMillis, 0);
+ }
+
+ public ContainerReplicaOp(PendingOpType opType,
+ DatanodeDetails target, int replicaIndex, SCMCommand<?> command,
+ long deadlineEpochMillis, long containerSize) {
this.opType = opType;
this.target = target;
this.replicaIndex = replicaIndex;
this.command = command;
this.deadlineEpochMillis = deadlineEpochMillis;
+ this.containerSize = containerSize;
}
public PendingOpType getOpType() {
@@ -65,6 +75,10 @@ public long getDeadlineEpochMillis() {
return deadlineEpochMillis;
}
+ public long getContainerSize() {
+ return containerSize;
+ }
+
/**
* Enum representing different types of pending Ops.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java
index d2d4389d737..8b1766cd978 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java
@@ -34,6 +34,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -59,12 +60,49 @@ public class ContainerReplicaPendingOps {
private ReplicationManagerMetrics replicationMetrics = null;
private final List<ContainerReplicaPendingOpsSubscriber> subscribers =
new ArrayList<>();
+ // tracks how much data is pending to be added to a target Datanode because of pending ADD ops
+ private final ConcurrentHashMap<DatanodeID, SizeAndTime> containerSizeScheduled = new ConcurrentHashMap<>();
+ private ReplicationManager.ReplicationManagerConfiguration rmConf;
public ContainerReplicaPendingOps(Clock clock) {
this.clock = clock;
resetCounters();
}
+ public ContainerReplicaPendingOps(Clock clock, ReplicationManager.ReplicationManagerConfiguration rmConf) {
+ this(clock);
+ this.rmConf = rmConf;
+ }
+
+ /**
+ * Used as the value of {@link ContainerReplicaPendingOps#containerSizeScheduled} map for tracking the size of
+ * containers with pending ADD ops. Immutable.
+ */
+ public static class SizeAndTime {
+ // number of bytes pending ADD on a target DN
+ private final long size;
+ // timestamp (milliseconds since epoch) when the latest op was scheduled for this DN
+ private final long lastUpdatedTime;
+
+ public SizeAndTime(long size, long lastUpdatedTime) {
+ this.size = size;
+ this.lastUpdatedTime = lastUpdatedTime;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public long getLastUpdatedTime() {
+ return lastUpdatedTime;
+ }
+
+ @Override
+ public String toString() {
+ return "Size: " + size + ", lastUpdatedTime: " + lastUpdatedTime;
+ }
+ }
+
private void resetCounters() {
for (PendingOpType opType: PendingOpType.values()) {
AtomicLong[] counters = new AtomicLong[2];
@@ -85,6 +123,7 @@ public void clear() {
try {
pendingOps.clear();
resetCounters();
+ containerSizeScheduled.clear();
} finally {
globalLock.writeLock().unlock();
}
@@ -123,8 +162,10 @@ public List<ContainerReplicaOp> getPendingOps(ContainerID containerID) {
* be discarded.
*/
public void scheduleAddReplica(ContainerID containerID,
- DatanodeDetails target, int replicaIndex, SCMCommand<?> command, long deadlineEpochMillis) {
- addReplica(ADD, containerID, target, replicaIndex, command, deadlineEpochMillis);
+ DatanodeDetails target, int replicaIndex, SCMCommand<?> command, long deadlineEpochMillis, long containerSize,
+ long scheduledEpochMillis) {
+ addReplica(ADD, containerID, target, replicaIndex, command, deadlineEpochMillis, containerSize,
+ scheduledEpochMillis);
}
/**
@@ -139,7 +180,7 @@ public void scheduleAddReplica(ContainerID containerID,
*/
public void scheduleDeleteReplica(ContainerID containerID,
DatanodeDetails target, int replicaIndex, SCMCommand<?> command, long deadlineEpochMillis) {
- addReplica(DELETE, containerID, target, replicaIndex, command, deadlineEpochMillis);
+ addReplica(DELETE, containerID, target, replicaIndex, command, deadlineEpochMillis, 0L, clock.millis());
}
/**
@@ -226,6 +267,9 @@ public void removeExpiredEntries() {
// For delete ops, we don't remove them from the list as RM must resend them, or they
// will be removed via a container report when they are confirmed as deleted.
iterator.remove();
+ if (op.getOpType() == ADD) {
+ releaseScheduledContainerSize(op);
+ }
decrementCounter(op.getOpType(), op.getReplicaIndex());
}
expiredOps.add(op);
@@ -246,6 +290,22 @@ public void removeExpiredEntries() {
}
}
+ private void releaseScheduledContainerSize(ContainerReplicaOp op) {
+ containerSizeScheduled.computeIfPresent(op.getTarget().getID(), (k, v) -> {
+ long newSize = v.getSize() - op.getContainerSize();
+ boolean isSizeNonPositive = newSize <= 0;
+ boolean hasOpExpired = clock.millis() - v.getLastUpdatedTime() > rmConf.getEventTimeout();
+ if (isSizeNonPositive || hasOpExpired) {
+ /*
+ If the scheduled size is now less than or equal to 0, or if the last op has expired, implying that the ops
+ before it must have completed or expired, then remove this entry from the map
+ */
+ return null;
+ }
+ return new SizeAndTime(newSize, v.getLastUpdatedTime());
+ });
+ }
+
private void updateTimeoutMetrics(ContainerReplicaOp op) {
if (op.getOpType() == ADD && isMetricsNotNull()) {
if (isEC(op.getReplicaIndex())) {
@@ -262,9 +322,10 @@ private void updateTimeoutMetrics(ContainerReplicaOp op) {
}
}
+ @SuppressWarnings("checkstyle:ParameterNumber")
private void addReplica(ContainerReplicaOp.PendingOpType opType,
ContainerID containerID, DatanodeDetails target, int replicaIndex, SCMCommand<?> command,
- long deadlineEpochMillis) {
+ long deadlineEpochMillis, long containerSize, long scheduledEpochMillis) {
Lock lock = writeLock(containerID);
lock(lock);
try {
@@ -274,7 +335,17 @@ private void addReplica(ContainerReplicaOp.PendingOpType opType,
List<ContainerReplicaOp> ops = pendingOps.computeIfAbsent(
containerID, s -> new ArrayList<>());
ops.add(new ContainerReplicaOp(opType,
- target, replicaIndex, command, deadlineEpochMillis));
+ target, replicaIndex, command, deadlineEpochMillis, containerSize));
+ DatanodeID id = target.getID();
+ if (opType == ADD) {
+ containerSizeScheduled.compute(id, (k, v) -> {
+ if (v == null) {
+ return new SizeAndTime(containerSize, scheduledEpochMillis);
+ } else {
+ return new SizeAndTime(v.getSize() + containerSize, scheduledEpochMillis);
+ }
+ });
+ }
incrementCounter(opType, replicaIndex);
} finally {
unlock(lock);
@@ -300,6 +371,15 @@ private boolean completeOp(ContainerReplicaOp.PendingOpType opType,
found = true;
completedOps.add(op);
iterator.remove();
+ if (opType == ADD) {
+ containerSizeScheduled.computeIfPresent(target.getID(), (k, v) -> {
+ long newSize = v.getSize() - op.getContainerSize();
+ if (newSize <= 0) {
+ return null;
+ }
+ return new SizeAndTime(newSize, v.getLastUpdatedTime());
+ });
+ }
decrementCounter(op.getOpType(), replicaIndex);
}
}
@@ -364,6 +444,14 @@ private void unlock(Lock lock) {
lock.unlock();
}
+ public ConcurrentHashMap<DatanodeID, SizeAndTime> getContainerSizeScheduled() {
+ return containerSizeScheduled;
+ }
+
+ public Clock getClock() {
+ return clock;
+ }
+
private boolean isMetricsNotNull() {
return replicationMetrics != null;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
index 89259cec08b..33ab37a9387 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
@@ -84,6 +84,7 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
import org.apache.hadoop.ozone.container.replication.ReplicationServer;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
@@ -205,7 +206,8 @@ public class ReplicationManager implements SCMService, ContainerReplicaPendingOp
* @param replicaPendingOps The pendingOps instance
*/
@SuppressWarnings("parameternumber")
- public ReplicationManager(final ConfigurationSource conf,
+ public ReplicationManager(final ReplicationManagerConfiguration rmConf,
+ final ConfigurationSource conf,
final ContainerManager containerManager,
final PlacementPolicy ratisContainerPlacement,
final PlacementPolicy ecContainerPlacement,
@@ -217,7 +219,7 @@ public ReplicationManager(final ConfigurationSource conf,
throws IOException {
this.containerManager = containerManager;
this.scmContext = scmContext;
- this.rmConf = conf.getObject(ReplicationManagerConfiguration.class);
+ this.rmConf = rmConf;
this.replicationServerConf =
conf.getObject(ReplicationServer.ReplicationConfig.class);
this.running = false;
@@ -688,13 +690,15 @@ private void adjustPendingOpsAndMetrics(ContainerInfo containerInfo,
ReconstructECContainersCommand rcc = (ReconstructECContainersCommand) cmd;
List<DatanodeDetails> targets = rcc.getTargetDatanodes();
final ByteString targetIndexes = rcc.getMissingContainerIndexes();
+ long requiredSize = HddsServerUtil.requiredReplicationSpace(containerInfo.getUsedBytes());
for (int i = 0; i < targetIndexes.size(); i++) {
- containerReplicaPendingOps.scheduleAddReplica(
- containerInfo.containerID(), targets.get(i), targetIndexes.byteAt(i), cmd, scmDeadlineEpochMs);
+ containerReplicaPendingOps.scheduleAddReplica(containerInfo.containerID(), targets.get(i),
+ targetIndexes.byteAt(i), cmd, scmDeadlineEpochMs, requiredSize, clock.millis());
}
getMetrics().incrEcReconstructionCmdsSentTotal();
} else if (cmd.getType() == Type.replicateContainerCommand) {
ReplicateContainerCommand rcc = (ReplicateContainerCommand) cmd;
+ long requiredSize = HddsServerUtil.requiredReplicationSpace(containerInfo.getUsedBytes());
if (rcc.getTargetDatanode() == null) {
/*
@@ -702,17 +706,15 @@ private void adjustPendingOpsAndMetrics(ContainerInfo containerInfo,
op's target Datanode should be the Datanode this command is being
sent to.
*/
- containerReplicaPendingOps.scheduleAddReplica(
- containerInfo.containerID(),
- targetDatanode, rcc.getReplicaIndex(), cmd, scmDeadlineEpochMs);
+ containerReplicaPendingOps.scheduleAddReplica(containerInfo.containerID(), targetDatanode,
+ rcc.getReplicaIndex(), cmd, scmDeadlineEpochMs, requiredSize, clock.millis());
} else {
/*
This means the source will push replica to the target, so the op's
target Datanode should be the Datanode the replica will be pushed to.
*/
- containerReplicaPendingOps.scheduleAddReplica(
- containerInfo.containerID(),
- rcc.getTargetDatanode(), rcc.getReplicaIndex(), cmd, scmDeadlineEpochMs);
+ containerReplicaPendingOps.scheduleAddReplica(containerInfo.containerID(), rcc.getTargetDatanode(),
+ rcc.getReplicaIndex(), cmd, scmDeadlineEpochMs, requiredSize, clock.millis());
}
if (rcc.getReplicaIndex() > 0) {
@@ -1484,6 +1486,10 @@ static NodeStatus getNodeStatus(DatanodeDetails dn, NodeManager nm) {
}
}
+ public NodeManager getNodeManager() {
+ return nodeManager;
+ }
+
private int getRemainingMaintenanceRedundancy(boolean isEC) {
return isEC
? rmConf.getMaintenanceRemainingRedundancy()
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java
index 267da521736..8b2ad758242 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java
@@ -24,6 +24,7 @@
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -34,7 +35,10 @@
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
@@ -195,9 +199,56 @@ An UNHEALTHY replica with unique origin node id of a QUASI_CLOSED container shou
excludedNodes.add(pending.getTarget());
}
}
+ excludeFullNodes(replicationManager, container, excludedNodes);
return new ExcludedAndUsedNodes(excludedNodes, usedNodes);
}
+ private static void excludeFullNodes(ReplicationManager replicationManager,
+ ContainerInfo container, List<DatanodeDetails> excludedNodes) {
+ ContainerReplicaPendingOps pendingOps = replicationManager.getContainerReplicaPendingOps();
+ Map<DatanodeID, ContainerReplicaPendingOps.SizeAndTime>
+ containerSizeScheduled = pendingOps.getContainerSizeScheduled();
+ if (containerSizeScheduled == null || containerSizeScheduled.isEmpty()) {
+ return;
+ }
+
+ final long requiredSize = HddsServerUtil.requiredReplicationSpace(container.getUsedBytes());
+ NodeManager nodeManager = replicationManager.getNodeManager();
+
+ for (Map.Entry<DatanodeID, ContainerReplicaPendingOps.SizeAndTime> entry : containerSizeScheduled.entrySet()) {
+ DatanodeDetails dn = nodeManager.getNode(entry.getKey());
+ if (dn == null || excludedNodes.contains(dn)) {
+ continue;
+ }
+
+ SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
+ if (nodeMetric == null) {
+ continue;
+ }
+
+ long scheduledSize = 0;
+ ContainerReplicaPendingOps.SizeAndTime sizeAndTime = entry.getValue();
+ if (sizeAndTime != null) {
+ // Only consider sizes added in the last event timeout window
+ if (pendingOps.getClock().millis() - sizeAndTime.getLastUpdatedTime()
+ < replicationManager.getConfig().getEventTimeout()) {
+ scheduledSize = sizeAndTime.getSize();
+ } else {
+ LOG.debug("Expired op {} found while computing exclude nodes", entry);
+ }
+ }
+
+ SCMNodeStat scmNodeStat = nodeMetric.get();
+ if (scmNodeStat.getRemaining().get() - scmNodeStat.getFreeSpaceToSpare().get() - scheduledSize
+ < requiredSize) {
+ LOG.debug("Adding datanode {} to exclude list. Remaining: {}, freeSpaceToSpare: {}, scheduledSize: {}, " +
+ "requiredSize: {}. ContainerInfo: {}.", dn, scmNodeStat.getRemaining().get(),
+ scmNodeStat.getFreeSpaceToSpare().get(), scheduledSize, requiredSize, container);
+ excludedNodes.add(dn);
+ }
+ }
+ }
+
/**
* Simple class to hold the excluded and used nodes lists.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECContainerSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECContainerSafeModeRule.java
index a6c30aace93..3ed27d8da8d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECContainerSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECContainerSafeModeRule.java
@@ -194,7 +194,6 @@ private void recordReportedContainer(long containerID) {
private void initializeRule() {
ecContainers.clear();
- ecContainerDNsMap.clear();
containerManager.getContainers(ReplicationType.EC).stream()
.filter(this::isClosed).filter(c -> c.getNumberOfKeys() > 0)
.map(ContainerInfo::getContainerID).forEach(ecContainers::add);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
index 95ac0f43d52..bc2a26fbf91 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
@@ -90,10 +90,10 @@ public SCMSafeModeManager(final ConfigurationSource conf,
this.scmContext = scmContext;
this.safeModeMetrics = SafeModeMetrics.create();
- // TODO: Remove the cyclic ("this") dependency (HDDS-11797)
- SafeModeRuleFactory.initialize(conf, scmContext, eventQueue, this,
+ SafeModeRuleFactory.initialize(conf, scmContext, eventQueue,
pipelineManager, containerManager, nodeManager);
SafeModeRuleFactory factory = SafeModeRuleFactory.getInstance();
+ factory.addSafeModeManager(this);
factory.getSafeModeRules().forEach(rule -> exitRules.put(rule.getRuleName(), rule));
factory.getPreCheckRules().forEach(rule -> preCheckRules.add(rule.getRuleName()));
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
index 9a7f1e2f9ec..6bd4dbf6444 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
@@ -35,8 +35,6 @@ public final class SafeModeRuleFactory {
private final SCMContext scmContext;
private final EventQueue eventQueue;
- // TODO: Remove dependency on safeModeManager (HDDS-11797)
- private final SCMSafeModeManager safeModeManager;
private final PipelineManager pipelineManager;
private final ContainerManager containerManager;
private final NodeManager nodeManager;
@@ -49,23 +47,24 @@ public final class SafeModeRuleFactory {
private SafeModeRuleFactory(final ConfigurationSource config,
final SCMContext scmContext,
final EventQueue eventQueue,
- final SCMSafeModeManager safeModeManager,
final PipelineManager pipelineManager,
final ContainerManager containerManager,
final NodeManager nodeManager) {
this.config = config;
this.scmContext = scmContext;
this.eventQueue = eventQueue;
- this.safeModeManager = safeModeManager;
this.pipelineManager = pipelineManager;
this.containerManager = containerManager;
this.nodeManager = nodeManager;
this.safeModeRules = new ArrayList<>();
this.preCheckRules = new ArrayList<>();
- loadRules();
}
- private void loadRules() {
+ public void addSafeModeManager(SCMSafeModeManager safeModeManager) {
+ loadRules(safeModeManager);
+ }
+
+ private void loadRules(SCMSafeModeManager safeModeManager) {
// TODO: Use annotation to load the rules. (HDDS-11730)
SafeModeExitRule<?> ratisContainerRule = new RatisContainerSafeModeRule(eventQueue,
config, containerManager, safeModeManager);
@@ -102,12 +101,11 @@ public static synchronized void initialize(
final ConfigurationSource config,
final SCMContext scmContext,
final EventQueue eventQueue,
- final SCMSafeModeManager safeModeManager,
final PipelineManager pipelineManager,
final ContainerManager containerManager,
final NodeManager nodeManager) {
instance = new SafeModeRuleFactory(config, scmContext, eventQueue,
- safeModeManager, pipelineManager, containerManager, nodeManager);
+ pipelineManager, containerManager, nodeManager);
}
public List<SafeModeExitRule<?>> getSafeModeRules() {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 5fa56be25fc..4f92c963bdf 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -775,8 +775,10 @@ private void initializeSystemManagers(OzoneConfiguration conf,
finalizationManager.buildUpgradeContext(scmNodeManager, pipelineManager,
scmContext);
+ ReplicationManager.ReplicationManagerConfiguration rmConf =
+ conf.getObject(ReplicationManager.ReplicationManagerConfiguration.class);
containerReplicaPendingOps =
- new ContainerReplicaPendingOps(systemClock);
+ new ContainerReplicaPendingOps(systemClock, rmConf);
long containerReplicaOpScrubberIntervalMs = conf.getTimeDuration(
ScmConfigKeys
@@ -829,6 +831,7 @@ private void initializeSystemManagers(OzoneConfiguration conf,
replicationManager = configurator.getReplicationManager();
} else {
replicationManager = new ReplicationManager(
+ rmConf,
conf,
containerManager,
containerPlacementPolicy,
@@ -838,7 +841,7 @@ private void initializeSystemManagers(OzoneConfiguration conf,
scmNodeManager,
systemClock,
containerReplicaPendingOps);
- reconfigurationHandler.register(replicationManager.getConfig());
+ reconfigurationHandler.register(rmConf);
}
serviceManager.register(replicationManager);
// RM gets notified of expired pending delete from containerReplicaPendingOps by subscribing to it
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index 0f15c9aa365..eba8a8eeb18 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -82,6 +82,8 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
/**
* Tests for DeletedBlockLog.
@@ -880,6 +882,41 @@ public void testDeletedBlockTransactions() throws IOException {
assertEquals(2, blocks.size());
}
+ @ParameterizedTest
+ @ValueSource(ints = {30, 45})
+ public void testGetTransactionsWithMaxBlocksPerDatanode(int maxAllowedBlockNum) throws IOException {
+ int deleteBlocksFactorPerDatanode = 1;
+ deletedBlockLog.setDeleteBlocksFactorPerDatanode(deleteBlocksFactorPerDatanode);
+ mockContainerHealthResult(true);
+ int txNum = 10;
+ DatanodeDetails dnId1 = dnList.get(0), dnId2 = dnList.get(1);
+
+ // Creates {TXNum} TX in the log.
+ Map<Long, List<Long>> deletedBlocks = generateData(txNum);
+ addTransactions(deletedBlocks, true);
+ List<Long> containerIds = new ArrayList<>(deletedBlocks.keySet());
+ for (int i = 0; i < containerIds.size(); i++) {
+ DatanodeDetails assignedDn = (i % 2 == 0) ? dnId1 : dnId2;
+ mockStandAloneContainerInfo(containerIds.get(i), assignedDn);
+ }
+
+ int blocksPerDataNode = maxAllowedBlockNum / (dnList.size() / deleteBlocksFactorPerDatanode);
+ DatanodeDeletedBlockTransactions transactions =
+ deletedBlockLog.getTransactions(maxAllowedBlockNum, new HashSet<>(dnList));
+
+ Map<DatanodeID, Integer> datanodeBlockCountMap = transactions.getDatanodeTransactionMap()
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue()
+ .stream()
+ .mapToInt(tx -> tx.getLocalIDList().size())
+ .sum()
+ ));
+ // Transactions should have blocksPerDataNode for both DNs
+ assertEquals(datanodeBlockCountMap.get(dnId1.getID()), blocksPerDataNode);
+ assertEquals(datanodeBlockCountMap.get(dnId2.getID()), blocksPerDataNode);
+ }
+
@Test
public void testDeletedBlockTransactionsOfDeletedContainer() throws IOException {
int txNum = 10;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java
index d645beb6240..ee813f0942c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java
@@ -22,6 +22,8 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -32,11 +34,12 @@
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
@@ -61,18 +64,20 @@ public class TestContainerReplicaPendingOps {
private long deadline;
private SCMCommand<?> addCmd;
private SCMCommand<?> deleteCmd;
+ private static final long FIVE_GB_CONTAINER_SIZE = 5L * 1024 * 1024 * 1024;
+ private static final long THREE_GB_CONTAINER_SIZE = 3L * 1024 * 1024 * 1024;
+ private ReplicationManager.ReplicationManagerConfiguration rmConf;
@BeforeEach
public void setup() {
clock = new TestClock(Instant.now(), ZoneOffset.UTC);
deadline = clock.millis() + 10000; // Current time plus 10 seconds
- pendingOps = new ContainerReplicaPendingOps(clock);
- ConfigurationSource conf = new OzoneConfiguration();
- ReplicationManager.ReplicationManagerConfiguration rmConf = conf
- .getObject(ReplicationManager.ReplicationManagerConfiguration.class);
+ OzoneConfiguration conf = new OzoneConfiguration();
+ rmConf = conf.getObject(ReplicationManager.ReplicationManagerConfiguration.class);
ReplicationManager rm = mock(ReplicationManager.class);
when(rm.getConfig()).thenReturn(rmConf);
+ pendingOps = new ContainerReplicaPendingOps(clock, rmConf);
metrics = ReplicationManagerMetrics.create(rm);
pendingOps.setReplicationMetrics(metrics);
dn1 = MockDatanodeDetails.randomDatanodeDetails();
@@ -99,7 +104,8 @@ public void testGetPendingOpsReturnsEmptyList() {
@Test
public void testClear() {
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(2), dn1, 0, deleteCmd, deadline);
assertEquals(1, pendingOps.getPendingOpCount(ContainerReplicaOp.PendingOpType.ADD));
@@ -116,16 +122,25 @@ public void testClear() {
@Test
public void testCanAddReplicasForAdd() {
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn2, 0, addCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn2, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
// Duplicate for DN2
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn2, 0, addCmd, deadline + 1);
+ pendingOps.scheduleAddReplica(
+ ContainerID.valueOf(1), dn2, 0, addCmd, deadline + 1,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
// Not a duplicate for DN2 as different index. Should not happen in practice as it is not valid to have 2 indexes
// on the same node.
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn2, 1, addCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 1, addCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 1, addCmd, deadline + 1);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn2, 1, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 1, addCmd, deadline,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(
+ ContainerID.valueOf(2), dn1, 1, addCmd, deadline + 1,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
List<ContainerReplicaOp> ops =
pendingOps.getPendingOps(ContainerID.valueOf(1));
@@ -185,9 +200,11 @@ public void testCanAddReplicasForDelete() {
@Test
public void testCompletingOps() {
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn1, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn2, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(2), dn1, 1, deleteCmd, deadline);
List<ContainerReplicaOp> ops =
@@ -218,9 +235,11 @@ public void testCompletingOps() {
@Test
public void testRemoveSpecificOp() {
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn1, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn2, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, deadline,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(2), dn1, 1, deleteCmd, deadline);
ContainerID cid = ContainerID.valueOf(1);
@@ -241,11 +260,14 @@ public void testRemoveExpiredEntries() {
long laterExpiry = clock.millis() + 2000;
long latestExpiry = clock.millis() + 3000;
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn1, 0, deleteCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, expiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd, expiry,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn2, 0, deleteCmd, laterExpiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, laterExpiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn3, 0, addCmd, laterExpiry,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(2), dn1, 1, deleteCmd, latestExpiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 1, addCmd, latestExpiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 1, addCmd, latestExpiry,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
List<ContainerReplicaOp> ops =
pendingOps.getPendingOps(ContainerID.valueOf(1));
@@ -302,10 +324,13 @@ private boolean isOpPresent(List<ContainerReplicaOp> ops, DatanodeDetails dn,
public void testReplicationMetrics() {
long expiry = clock.millis() + 1000;
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(1), dn1, 1, deleteCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 2, addCmd, expiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 2, addCmd, expiry,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(2), dn2, 1, deleteCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn3, 1, addCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(3), dn3, 0, addCmd, expiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn3, 1, addCmd, expiry,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(3), dn3, 0, addCmd, expiry,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(4), dn3, 0, deleteCmd, expiry);
// InFlight Replication and Deletion
@@ -328,10 +353,13 @@ public void testReplicationMetrics() {
expiry = clock.millis() + 1000;
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(3), dn1, 2, deleteCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(3), dn1, 3, addCmd, expiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(3), dn1, 3, addCmd, expiry,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(4), dn2, 2, deleteCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(4), dn3, 4, addCmd, expiry);
- pendingOps.scheduleAddReplica(ContainerID.valueOf(5), dn3, 0, addCmd, expiry);
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(4), dn3, 4, addCmd, expiry,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(5), dn3, 0, addCmd, expiry,
+ THREE_GB_CONTAINER_SIZE, clock.millis());
pendingOps.scheduleDeleteReplica(ContainerID.valueOf(6), dn3, 0, deleteCmd, expiry);
// InFlight Replication and Deletion. Previous Inflight should be
@@ -375,7 +403,7 @@ public void testNotifySubscribers() {
// schedule an ADD and a DELETE
ContainerID containerID = ContainerID.valueOf(1);
- pendingOps.scheduleAddReplica(containerID, dn1, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(containerID, dn1, 0, addCmd, deadline, FIVE_GB_CONTAINER_SIZE, clock.millis());
ContainerReplicaOp addOp = pendingOps.getPendingOps(containerID).get(0);
pendingOps.scheduleDeleteReplica(containerID, dn1, 0, deleteCmd, deadline);
@@ -392,7 +420,7 @@ public void testNotifySubscribers() {
// now, test notification on expiration
pendingOps.scheduleDeleteReplica(containerID, dn1, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline, FIVE_GB_CONTAINER_SIZE, clock.millis());
for (ContainerReplicaOp op : pendingOps.getPendingOps(containerID)) {
if (op.getOpType() == ADD) {
addOp = op;
@@ -415,7 +443,7 @@ public void subscribersShouldNotBeNotifiedWhenOpsHaveNotExpired() {
// schedule ops
pendingOps.scheduleDeleteReplica(containerID, dn1, 0, deleteCmd, deadline);
- pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline, FIVE_GB_CONTAINER_SIZE, clock.millis());
// register subscriber
ContainerReplicaPendingOpsSubscriber subscriber1 = mock(
@@ -434,7 +462,7 @@ public void subscribersShouldNotBeNotifiedWhenReplacingAnOpWithDuplicate() {
ContainerID containerID = ContainerID.valueOf(1);
// schedule ops
- pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline);
+ pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline, FIVE_GB_CONTAINER_SIZE, clock.millis());
// register subscriber
ContainerReplicaPendingOpsSubscriber subscriber1 = mock(
@@ -442,9 +470,116 @@ public void subscribersShouldNotBeNotifiedWhenReplacingAnOpWithDuplicate() {
pendingOps.registerSubscriber(subscriber1);
clock.fastForward(1000);
- pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline + 1);
+ pendingOps.scheduleAddReplica(containerID, dn2, 0, addCmd, deadline + 1,
+ FIVE_GB_CONTAINER_SIZE, clock.millis());
// no entries have expired, so there should be zero interactions with the
// subscriber
verifyNoMoreInteractions(subscriber1);
}
+
+ /**
+ * Tests that ContainerReplicaPendingOps correctly tracks how much size (of containers) is being moved to a target
+ * Datanode because of pending ADD ops. This size should be correctly added and reduced when ADD ops are triggered
+ * and completed.
+ */
+ @Test
+ public void testScheduledSizeIsCorrectlyTrackedAndCompleted() {
+ final long eventTimeout = rmConf.getEventTimeout();
+ long now = clock.millis();
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(1), dn1, 0, addCmd,
+ now + eventTimeout, FIVE_GB_CONTAINER_SIZE, clock.millis());
+
+ // Assert that containerSizeScheduled has the correct size
+ ConcurrentHashMap<DatanodeID, ContainerReplicaPendingOps.SizeAndTime> scheduled =
+ pendingOps.getContainerSizeScheduled();
+ assertEquals(1, scheduled.size());
+ assertEquals(FIVE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+
+ // Schedule a second op for the same datanode
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(2), dn1, 0, addCmd,
+ now + eventTimeout, THREE_GB_CONTAINER_SIZE, clock.millis());
+ assertEquals(FIVE_GB_CONTAINER_SIZE + THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+
+ // Complete the first op
+ pendingOps.completeAddReplica(ContainerID.valueOf(1), dn1, 0);
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+
+ // Complete the second op
+ pendingOps.completeAddReplica(ContainerID.valueOf(2), dn1, 0);
+ assertNull(scheduled.get(dn1.getID()));
+ }
+
+ /**
+ * When an ADD op (container replication) expires, the map in ContainerReplicaPendingOps should be modified
+ * correctly. The entry should be removed if ReplicationManagerConfiguration#eventTimemout milliseconds have passed
+ * since the entry's lastUpdatedTime.
+ */
+ @Test
+ public void testScheduledSizeIsCorrectlyTrackedAndExpired() {
+ final long eventTimeout = rmConf.getEventTimeout();
+
+ long now = clock.millis();
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(3), dn2, 0, addCmd,
+ now + eventTimeout, FIVE_GB_CONTAINER_SIZE, clock.millis());
+ ConcurrentHashMap<DatanodeID, ContainerReplicaPendingOps.SizeAndTime>
+ scheduled = pendingOps.getContainerSizeScheduled();
+ assertEquals(FIVE_GB_CONTAINER_SIZE, scheduled.get(dn2.getID()).getSize());
+ assertEquals(now, scheduled.get(dn2.getID()).getLastUpdatedTime());
+
+ // Advance clock so the op expires
+ clock.fastForward(eventTimeout + 1);
+ pendingOps.removeExpiredEntries();
+ // The entry should be removed from the map after expiration
+ assertNull(scheduled.get(dn2.getID()));
+ }
+
+ /**
+ * Tests that only the size of containers with expired ops is reduced from the map tracking size of pending ops.
+ * For example, if target Datanode DN1 has two pending ADD ops 10GB + 15GB, and the first op expires, then only
+ * 10GB should be subtracted.
+ */
+ @Test
+ public void testOnlyExpiredOpSizeIsRemovedFromSizeScheduledMap() {
+ final long eventTimeout = rmConf.getEventTimeout();
+ long now = clock.millis();
+ // Schedule first op
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(4), dn2, 0, addCmd,
+ now + eventTimeout, FIVE_GB_CONTAINER_SIZE, clock.millis());
+ // another replication scheduled for dn1 to receive a container - just testing that this entry isn't removed or
+ // modified when other entries expire or are modified
+ pendingOps.scheduleAddReplica((ContainerID.valueOf(2)), dn1, 2, addCmd,
+ now + eventTimeout * 10, THREE_GB_CONTAINER_SIZE, clock.millis());
+ ConcurrentHashMap<DatanodeID, ContainerReplicaPendingOps.SizeAndTime>
+ scheduled = pendingOps.getContainerSizeScheduled();
+ assertEquals(FIVE_GB_CONTAINER_SIZE, scheduled.get(dn2.getID()).getSize());
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+ assertEquals(now, scheduled.get(dn2.getID()).getLastUpdatedTime());
+ assertEquals(now, scheduled.get(dn1.getID()).getLastUpdatedTime());
+
+ clock.fastForward(eventTimeout - 1);
+ long updateTime = clock.millis();
+
+ // Schedule second op for dn2, which should update the lastUpdatedTime
+ pendingOps.scheduleAddReplica(ContainerID.valueOf(5), dn2, 1, addCmd,
+ updateTime + eventTimeout, THREE_GB_CONTAINER_SIZE, clock.millis());
+ assertEquals(FIVE_GB_CONTAINER_SIZE + THREE_GB_CONTAINER_SIZE, scheduled.get(dn2.getID()).getSize());
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+ assertEquals(updateTime, scheduled.get(dn2.getID()).getLastUpdatedTime());
+ assertEquals(now, scheduled.get(dn1.getID()).getLastUpdatedTime());
+
+ // Advance clock to expire the first op but not the second for dn2
+ clock.set(Instant.ofEpochMilli(now + eventTimeout + 1));
+ pendingOps.removeExpiredEntries();
+
+ // Assert the entry for dn2 still exists, but with reduced size
+ assertNotNull(scheduled.get(dn2.getID()));
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn2.getID()).getSize());
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+
+ // Advance clock again to expire the second op for dn2
+ clock.set(Instant.ofEpochMilli(updateTime + eventTimeout + 1));
+ pendingOps.removeExpiredEntries();
+ assertNull(scheduled.get(dn2.getID()));
+ assertEquals(THREE_GB_CONTAINER_SIZE, scheduled.get(dn1.getID()).getSize());
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java
index 8a18c1ba1dd..5de354fb2de 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java
@@ -141,6 +141,7 @@ public NodeStatus getNodeStatus(DatanodeDetails dd) {
.thenReturn(rmConf);
metrics = ReplicationManagerMetrics.create(replicationManager);
when(replicationManager.getMetrics()).thenReturn(metrics);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(mock(ContainerReplicaPendingOps.class));
when(replicationManager.getNodeStatus(any(DatanodeDetails.class)))
.thenAnswer(invocation -> {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java
index 59ffab79c3d..d65cd2afc08 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java
@@ -94,6 +94,7 @@ protected void setup(ReplicationConfig repConfig, File testDir)
when(replicationManager.getConfig()).thenReturn(rmConf);
metrics = ReplicationManagerMetrics.create(replicationManager);
when(replicationManager.getMetrics()).thenReturn(metrics);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(mock(ContainerReplicaPendingOps.class));
commandsSent = new HashSet<>();
ReplicationTestUtil.mockRMSendDatanodeCommand(
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
index d88bcf86af9..22fee58b7c4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
@@ -88,6 +88,7 @@ void setup(@TempDir File testDir) throws NodeNotFoundException,
ReplicationManager.ReplicationManagerConfiguration.class));
metrics = ReplicationManagerMetrics.create(replicationManager);
when(replicationManager.getMetrics()).thenReturn(metrics);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(mock(ContainerReplicaPendingOps.class));
/*
Return NodeStatus with NodeOperationalState as specified in
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java
index 21bc259b1a2..dd2f9fd51d0 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java
@@ -106,6 +106,7 @@ void setup(@TempDir File testDir) throws NodeNotFoundException,
ReplicationManagerConfiguration.class));
metrics = ReplicationManagerMetrics.create(replicationManager);
when(replicationManager.getMetrics()).thenReturn(metrics);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(mock(ContainerReplicaPendingOps.class));
/*
Return NodeStatus with NodeOperationalState as specified in
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
index 349da3dcaa5..204953a6033 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
@@ -48,6 +48,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Lists;
import com.google.protobuf.Proto2Utils;
import java.io.IOException;
import java.time.Instant;
@@ -98,7 +99,6 @@
import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.util.Lists;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.TestClock;
import org.apache.ratis.protocol.exceptions.NotLeaderException;
@@ -201,6 +201,7 @@ void cleanup() {
private ReplicationManager createReplicationManager() throws IOException {
return new ReplicationManager(
+ configuration.getObject(ReplicationManager.ReplicationManagerConfiguration.class),
configuration,
containerManager,
ratisPlacementPolicy,
@@ -226,7 +227,7 @@ private void enableProcessAll() {
@Test
public void testPendingOpsClearedWhenStarting() {
containerReplicaPendingOps.scheduleAddReplica(ContainerID.valueOf(1),
- MockDatanodeDetails.randomDatanodeDetails(), 1, null, Integer.MAX_VALUE);
+ MockDatanodeDetails.randomDatanodeDetails(), 1, null, Integer.MAX_VALUE, 5L, clock.millis());
containerReplicaPendingOps.scheduleDeleteReplica(ContainerID.valueOf(2),
MockDatanodeDetails.randomDatanodeDetails(), 1, null, Integer.MAX_VALUE);
assertEquals(1, containerReplicaPendingOps
@@ -618,7 +619,7 @@ public void testUnderReplicatedContainerFixedByPending()
addReplicas(container, ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4);
containerReplicaPendingOps.scheduleAddReplica(container.containerID(),
MockDatanodeDetails.randomDatanodeDetails(), 5, null,
- clock.millis() + 10000);
+ clock.millis() + 10000, 5L, clock.millis());
replicationManager.processContainer(
container, repQueue, repReport);
@@ -1660,6 +1661,7 @@ public void testNotifyNodeStateChangeWakesUpThread(boolean queueIsEmpty)
ReplicationQueue queue = mock(ReplicationQueue.class);
when(queue.isEmpty()).thenReturn(queueIsEmpty);
final ReplicationManager customRM = new ReplicationManager(
+ configuration.getObject(ReplicationManager.ReplicationManagerConfiguration.class),
configuration,
containerManager,
ratisPlacementPolicy,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerScenarios.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerScenarios.java
index c0315738ae2..e101bf6ae59 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerScenarios.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerScenarios.java
@@ -222,6 +222,7 @@ public void setup() throws IOException, NodeNotFoundException {
private ReplicationManager createReplicationManager() throws IOException {
return new ReplicationManager(
+ configuration.getObject(ReplicationManager.ReplicationManagerConfiguration.class),
configuration,
containerManager,
ratisPlacementPolicy,
@@ -250,7 +251,7 @@ private void loadPendingOps(ContainerInfo container, Scenario scenario) {
for (PendingReplica r : scenario.getPendingReplicas()) {
if (r.getType() == ContainerReplicaOp.PendingOpType.ADD) {
containerReplicaPendingOps.scheduleAddReplica(container.containerID(), r.getDatanodeDetails(),
- r.getReplicaIndex(), null, Long.MAX_VALUE);
+ r.getReplicaIndex(), null, Long.MAX_VALUE, 5L, clock.millis());
} else if (r.getType() == ContainerReplicaOp.PendingOpType.DELETE) {
containerReplicaPendingOps.scheduleDeleteReplica(container.containerID(), r.getDatanodeDetails(),
r.getReplicaIndex(), null, Long.MAX_VALUE);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerUtil.java
index 97abe6be99e..a7718bdf24c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerUtil.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerUtil.java
@@ -20,28 +20,38 @@
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps.SizeAndTime;
import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.createContainer;
import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.createContainerReplica;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.ozone.test.TestClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -55,6 +65,8 @@ public class TestReplicationManagerUtil {
@BeforeEach
public void setup() {
replicationManager = mock(ReplicationManager.class);
+ ContainerReplicaPendingOps pendingOpsMock = mock(ContainerReplicaPendingOps.class);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(pendingOpsMock);
}
@Test
@@ -222,4 +234,114 @@ public void testGetUsedAndExcludedNodesForQuasiClosedContainer() throws NodeNotF
.contains(pendingDelete);
}
+ @Test
+ public void testDatanodesWithInSufficientDiskSpaceAreExcluded() throws NodeNotFoundException {
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
+ final long oneGb = 1024L * 1024 * 1024;
+ container.setUsedBytes(5 * oneGb);
+ ContainerID cid = container.containerID();
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ContainerReplica good = createContainerReplica(cid, 0,
+ IN_SERVICE, ContainerReplicaProto.State.CLOSED, 1);
+ replicas.add(good);
+
+ ContainerReplica remove = createContainerReplica(cid, 0,
+ IN_SERVICE, ContainerReplicaProto.State.CLOSED, 1);
+ replicas.add(remove);
+
+ ContainerReplica unhealthy = createContainerReplica(
+ cid, 0, IN_SERVICE, ContainerReplicaProto.State.UNHEALTHY, 1);
+ replicas.add(unhealthy);
+
+ ContainerReplica decommissioning =
+ createContainerReplica(cid, 0,
+ DECOMMISSIONING, ContainerReplicaProto.State.CLOSED, 1);
+ replicas.add(decommissioning);
+
+ ContainerReplica maintenance =
+ createContainerReplica(cid, 0,
+ IN_MAINTENANCE, ContainerReplicaProto.State.CLOSED, 1);
+ replicas.add(maintenance);
+
+ // Take one of the replicas and set it to be removed. It should be on the
+ // excluded list rather than the used list.
+ Set<ContainerReplica> toBeRemoved = new HashSet<>();
+ toBeRemoved.add(remove);
+
+ // Add a pending add and delete. The add should go onto the used
+ // list and the delete added to the excluded nodes.
+ DatanodeDetails pendingAdd = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails pendingDelete = MockDatanodeDetails.randomDatanodeDetails();
+ List<ContainerReplicaOp> pending = new ArrayList<>();
+ pending.add(ContainerReplicaOp.create(
+ ContainerReplicaOp.PendingOpType.ADD, pendingAdd, 0));
+ pending.add(ContainerReplicaOp.create(
+ ContainerReplicaOp.PendingOpType.DELETE, pendingDelete, 0));
+
+ // set up mocks such ContainerReplicaPendingOps returns the containerSizeScheduled map
+ ReplicationManagerConfiguration rmConf = new ReplicationManagerConfiguration();
+ when(replicationManager.getConfig()).thenReturn(rmConf);
+ TestClock clock = new TestClock(Instant.now(), ZoneOffset.UTC);
+ ConcurrentHashMap<DatanodeID, SizeAndTime> sizeScheduledMap = new ConcurrentHashMap<>();
+
+ // fullDn has 10GB size scheduled, 30GB available and 20GB min free space, so it should be excluded
+ DatanodeDetails fullDn = MockDatanodeDetails.randomDatanodeDetails();
+ sizeScheduledMap.put(fullDn.getID(), new SizeAndTime(10 * oneGb, clock.millis()));
+ // spaceAvailableDn should not be excluded as it has sufficient space
+ DatanodeDetails spaceAvailableDn = MockDatanodeDetails.randomDatanodeDetails();
+ sizeScheduledMap.put(spaceAvailableDn.getID(), new SizeAndTime(10 * oneGb, clock.millis()));
+ // expiredOpDn is the same as fullDn, however its op has expired - so it should not be excluded
+ DatanodeDetails expiredOpDn = MockDatanodeDetails.randomDatanodeDetails();
+ sizeScheduledMap.put(expiredOpDn.getID(), new SizeAndTime(10 * oneGb,
+ clock.millis() - rmConf.getEventTimeout() - 1));
+
+ ContainerReplicaPendingOps pendingOpsMock = mock(ContainerReplicaPendingOps.class);
+ when(pendingOpsMock.getContainerSizeScheduled()).thenReturn(sizeScheduledMap);
+ when(pendingOpsMock.getClock()).thenReturn(clock);
+ when(replicationManager.getContainerReplicaPendingOps()).thenReturn(pendingOpsMock);
+
+ NodeManager nodeManagerMock = mock(NodeManager.class);
+ when(replicationManager.getNodeManager()).thenReturn(nodeManagerMock);
+ doReturn(fullDn).when(nodeManagerMock).getNode(fullDn.getID());
+ doReturn(new SCMNodeMetric(50 * oneGb, 20 * oneGb, 30 * oneGb, 5 * oneGb,
+ 20 * oneGb)).when(nodeManagerMock).getNodeStat(fullDn);
+ doReturn(spaceAvailableDn).when(nodeManagerMock).getNode(spaceAvailableDn.getID());
+ doReturn(new SCMNodeMetric(50 * oneGb, 10 * oneGb, 40 * oneGb, 5 * oneGb,
+ 20 * oneGb)).when(nodeManagerMock).getNodeStat(spaceAvailableDn);
+ doReturn(expiredOpDn).when(nodeManagerMock).getNode(expiredOpDn.getID());
+ doReturn(new SCMNodeMetric(50 * oneGb, 20 * oneGb, 30 * oneGb, 5 * oneGb,
+ 20 * oneGb)).when(nodeManagerMock).getNodeStat(expiredOpDn);
+
+ when(replicationManager.getNodeStatus(any())).thenAnswer(
+ invocation -> {
+ final DatanodeDetails dn = invocation.getArgument(0);
+ for (ContainerReplica r : replicas) {
+ if (r.getDatanodeDetails().equals(dn)) {
+ return NodeStatus.valueOf(
+ r.getDatanodeDetails().getPersistedOpState(),
+ HddsProtos.NodeState.HEALTHY);
+ }
+ }
+ throw new NodeNotFoundException(dn.getID());
+ });
+
+ ReplicationManagerUtil.ExcludedAndUsedNodes excludedAndUsedNodes =
+ ReplicationManagerUtil.getExcludedAndUsedNodes(container,
+ new ArrayList<>(replicas), toBeRemoved, pending,
+ replicationManager);
+
+ assertEquals(3, excludedAndUsedNodes.getUsedNodes().size());
+ assertThat(excludedAndUsedNodes.getUsedNodes()).contains(good.getDatanodeDetails());
+ assertThat(excludedAndUsedNodes.getUsedNodes()).contains(maintenance.getDatanodeDetails());
+ assertThat(excludedAndUsedNodes.getUsedNodes()).contains(pendingAdd);
+
+ assertEquals(5, excludedAndUsedNodes.getExcludedNodes().size());
+ assertThat(excludedAndUsedNodes.getExcludedNodes()).contains(unhealthy.getDatanodeDetails());
+ assertThat(excludedAndUsedNodes.getExcludedNodes()).contains(decommissioning.getDatanodeDetails());
+ assertThat(excludedAndUsedNodes.getExcludedNodes()).contains(remove.getDatanodeDetails());
+ assertThat(excludedAndUsedNodes.getExcludedNodes()).contains(pendingDelete);
+ assertThat(excludedAndUsedNodes.getExcludedNodes()).contains(fullDn);
+ }
+
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECContainerSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECContainerSafeModeRule.java
new file mode 100644
index 00000000000..aceff644ec3
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECContainerSafeModeRule.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.safemode;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+/**
+ * This class tests ECContainerSafeModeRule.
+ */
+public class TestECContainerSafeModeRule {
+ private ContainerManager containerManager;
+ private ConfigurationSource conf;
+ private EventQueue eventQueue;
+ private SCMSafeModeManager safeModeManager;
+ private SafeModeMetrics metrics;
+
+ private ECContainerSafeModeRule rule;
+
+ @BeforeEach
+ public void setup() {
+ containerManager = mock(ContainerManager.class);
+ conf = mock(ConfigurationSource.class);
+ eventQueue = mock(EventQueue.class);
+ safeModeManager = mock(SCMSafeModeManager.class);
+ metrics = mock(SafeModeMetrics.class);
+
+ when(safeModeManager.getSafeModeMetrics()).thenReturn(metrics);
+
+ rule = new ECContainerSafeModeRule(eventQueue, conf, containerManager, safeModeManager);
+ rule.setValidateBasedOnReportProcessing(false);
+ }
+
+ @Test
+ public void testRefreshInitializeECContainers() {
+ List<ContainerInfo> containers = Arrays.asList(
+ mockECContainer(LifeCycleState.CLOSED, 1L),
+ mockECContainer(LifeCycleState.OPEN, 2L)
+ );
+
+ when(containerManager.getContainers(ReplicationType.EC)).thenReturn(containers);
+
+ rule.refresh(false);
+
... 16549 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org