You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by er...@apache.org on 2026/10/06 21:32:10 UTC
(ozone) branch HDDS-14496-zdu updated (c064bafb5e1 -> a9a64dbed31)
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a change to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
from c064bafb5e1 Merge branch 'master' into HDDS-14496-zdu
add 36481b61e44 HDDS-16595. Bump download-maven-plugin to 2.1.0 (#11316)
add 119fc06fd8f HDDS-16613. Bump actions/labeler to 7.0.0 (#11336)
add f39a349e90d HDDS-16623. Datanode can be missing from NetworkTopology after it recovers from DEAD (#11343)
add 4b020c16d65 HDDS-16597. Bump iceberg to 1.11.0 (#11318)
add 03d838e01e0 HDDS-16398. Fix intermittent AlreadyClosedException in TestCommitWatcher (#11227)
add 8bdfef0b8fb HDDS-13365. OM support automatic table cache tracking and cleanup (#10171)
add 7b7e453e9ab HDDS-16644. Bump jackson to 2.21.7 (#11352)
add ba05c2e3ad8 HDDS-16497. Support OM ThreadLocal context propagation for Ratis read (#11257)
add 2468fa8b14d HDDS-16647. Set only server-side target version for compile check (#11362)
add 907411afe71 HDDS-12950. Refactor TestBucketList to reduce duplicated declarations (#11338)
add 3448f66a4df HDDS-16645. Replace usage of deprecated AccessController in TestOmMixedWorkloadUnderDeletionBench (#11363)
add ce4953e56ed HDDS-16646. Remove listMaxSize from KeyLifecycleService. (#11361)
add 12635659b13 HDDS-16605. Reduce noise from failed copy-jar in integration check (#11326)
add 46a0ec802b9 HDDS-16551. Remove duplicate heatmap feature check in Recon v2 UI (#11323)
add b15c871db90 HDDS-16642. Hide heatmap navbar tab until feature status is confirmed in Recon v2 (#11360)
add db63057eaf2 HDDS-16635. Remove redundant InfoBucket RPC from S3 GetBucketLifecycleConfiguration (#11355)
add 8638daf7f3e HDDS-16636. Remove redundant InfoBucket RPC from S3 ListParts (#11358)
add 24e06a40c16 HDDS-15835. Count incomplete MPU parts in quota repair (#10795)
add 2b5874b743b HDDS-16226. Tool to obtain snapshot trapped bytes for a bucket. (#11122)
add 4317f21bc85 HDDS-16603. Remove redundant infoBucket RPC call in S3 tagging APIs (#11339)
add a0119aaebd2 HDDS-16637. Remove redundant InfoBucket RPC from S3 ListMultipartUploads (#11359)
add 0f20ca7b287 HDDS-16655. Skip shading in integration (filesystem) check (#11371)
add 8c249d0a048 HDDS-15142. Support HMAC-SHA256 trailer signature verification for S3 chunked uploads (#11222)
add 985d7a94e54 HDDS-16099. Speed up TestStreamBlockInputStream (#11337)
add ecafca6f6f6 HDDS-16630. Update the network topology when a datanode re-registers with new ports or a new version (#11353)
add d5d4df4d4b6 HDDS-16395. Add lock-free StreamBlock pread (#11314)
add 4a832912fea HDDS-16591. Reject not implemented S3 subresource APIs (#11313)
add f35c48c9f64 HDDS-8280. Upgrade Jetty 9.4 to 12.0.38 (EE8) (#11367)
add aade154f8dd HDDS-16648. Tune HTTP settings for CI (#11375)
add 29758e4ac1d HDDS-16441. Orphaned snapshot diff task marks a resubmitted job DONE with another run's metadata (#11321)
add 037a367ceb1 HDDS-16650. Reuse bucket metadata in S3 lifecycle PUT and DELETE (#11372)
add fd90ad1b7a5 HDDS-16530. Allow Recon service principal to call OM getDBUpdates (#11382).
add f7bfdf19b63 HDDS-16659. Reuse destination OzoneBucket in streaming S3 CopyObject (#11383)
add 63f93f7ca79 HDDS-16323. StreamBlockInputStream: refill the pre-read window in bulk instead of once per response (#11159)
add 60ce9947960 HDDS-16619. Intermittent failure in TestReconAsPassiveScm.testDatanodeRegistrationAndReports (#11341)
add 177fe51472f HDDS-13303. Fix TestSnapshotDeletingServiceIntegrationTest (#11310)
add 29b98532703 S3 header auth expiry in 15min and correct date validation.
add 03f73b4b07a HDDS-16705. Bump zstd-jni to 1.5.7-20 (#11390)
add ae281be3149 HDDS-16707. Bump commons-lang3 to 3.21.0 (#11394)
add 317c423af3e HDDS-16706. Bump slf4j to 2.0.20 (#11392)
add 225727d1890 HDDS-2821. Include datanode hostname/IP in logs in addition to UUID (#11381)
add 898bb4310f2 HDDS-14337. Fix SCM startup failure near year boundary when seeding localId (#11184)
add 7f7cb3f5597 HDDS-14018. Avoid repeated operations in SCMNodeManager#getNodeStatistics (#11292)
add 69c51c4c2ca HDDS-16710. Bump axios to 1.20.0 (#11378)
add 2e51e156477 HDDS-16711. Bump moment to 2.31.0 (#11373)
add ac9a57ddc1b HDDS-16658. Reuse source key details when opening input stream in S3 CopyObject (#11396)
add 4c99fd8fe12 HDDS-16654. Replace usage of deprecated finalize() in OM (#11376)
add 77d582975a6 HDDS-16631. Fix chooseRandom for rack names with common prefixes (#11401)
add 037f9161a33 HDDS-16704. ReadBlock fails with IllegalStateException when a response is shorter than responseDataSize (#11402)
add 65c4078829d HDDS-15089. Support S3 per request read consistency (#11252)
add ec64e924c40 HDDS-16300. Allow OM to dynamically reconfigure its SCM node list without a restart (#11218)
add 4d1e5d7b49e HDDS-16713. Bump develocity-maven-extension to 2.6.0 (#11405)
add 6772ee33dc3 HDDS-16708. Let dependabot ignore iceberg minor version upgrades (#11398)
add 3afe9a108f5 HDDS-16673. Avoid redundant ListBuckets RPCs when S3 bucket listing reaches the end (#11397)
add b79d76542ed HDDS-16674. Bump awssdk to 2.55.6 (#11407)
add 4066b0e8e23 HDDS-15991. Speed up deleted table scans in quota repair (#11386)
add 1ad6848c3c4 HDDS-16241. gRPC deadline kills long-lived block streams after 30 seconds and the client never recovers (#11080)
add 7ed017d22e4 HDDS-16721. StreamBlockInputStream.read() returns a negative value for bytes 0x80 to 0xFF (#11411)
add 646c4dfd8c1 HDDS-16643. Remove CleanupTableInfo mechanism (#11365)
add 96c43b3354b HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for FSO buckets (#11242).
add ccce9187dcb HDDS-15749. Run specific JUnit tests if possible (#10671)
add fb0adda14fc HDDS-16666. Flush SCM transaction in memory during apply transaction (#11409)
add 7d9e8417e83 HDDS-16008. PutBlocks from Flushes also go without Raft (#11356)
add a99fabb9dea HDDS-16716. Add description for ozone.scm.ec.pipeline.per.volume.factor (#11415)
new a9a64dbed31 Merge branch 'master' into HDDS-14496-zdu
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/dependabot.yml | 8 +-
.github/workflows/build-ratis.yml | 2 +-
.github/workflows/check.yml | 2 +-
.github/workflows/ci.yml | 35 +-
.github/workflows/intermittent-test-check.yml | 2 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
dev-support/ci/selective_ci_checks.bats | 42 +-
dev-support/ci/selective_ci_checks.sh | 68 +++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 5 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 77 +++-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hdds/scm/storage/MultipartInputStream.java | 48 +--
.../hadoop/hdds/scm/storage/PartInputStream.java | 13 +
.../hdds/scm/storage/StreamBlockInputStream.java | 182 +++++++--
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +-
.../hdds/scm/storage/MockDatanodePipeline.java | 45 +-
.../scm/storage/TestBlockDataStreamOutput.java | 59 ++-
.../hdds/scm/storage/TestMultipartInputStream.java | 69 ++++
.../scm/storage/TestStreamBlockInputStream.java | 361 ++++++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 23 +-
.../hdds/utils/TestConnectionFailureUtils.java | 14 +
.../common/statemachine/DatanodeConfiguration.java | 22 +
.../ClosePipelineCommandHandler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 68 +++-
.../common/transport/server/XceiverServerGrpc.java | 5 +-
.../server/ratis/ContainerStateMachine.java | 31 +-
.../transport/server/ratis/DispatcherContext.java | 4 +-
.../common/transport/server/ratis/LocalStream.java | 22 +-
.../container/keyvalue/ReadBlockComputation.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 3 +-
.../transport/server/TestGrpcXceiverService.java | 169 ++++++++
.../container/keyvalue/TestKeyValueHandler.java | 18 +
.../docs/content/feature/Reconfigurability.md | 2 +
hadoop-hdds/framework/pom.xml | 22 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 91 ++++-
.../hdds/server/http/AdminAuthorizedServlet.java | 2 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 56 ++-
...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 244 ++++++++---
.../hdds/server/http/ServletElementsFactory.java | 4 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +
.../utils/db/cache/TableCacheUpdateTracker.java | 88 ++++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 19 +
.../TestSCMFailoverProxyProviderChangeConfig.java | 149 +++++++
.../hdds/server/http/TestBaseHttpServer.java | 53 +++
.../hadoop/hdds/server/http/TestHttpServer2.java | 390 ++++++++++++++++++
.../hdds/server/http/TestHttpServer2SSL.java | 36 +-
.../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 ++++++
hadoop-hdds/rocks-native/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 12 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 2 +
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 41 ++
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 5 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 --
.../hadoop/hdds/scm/node/NodeStateManager.java | 32 +-
.../hdds/scm/node/PendingContainerTracker.java | 2 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 221 ++++++----
.../scm/node/UnhealthyToHealthyNodeHandler.java | 15 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 5 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++++++++++++++
.../ha/TestSCMHATransactionBufferMonitorTask.java | 1 +
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 124 +++++-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 53 +++
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 57 +--
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 88 +++-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 63 ++-
.../admin/om/BucketDeletedBytesSubCommand.java | 101 +++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../ozone/client/S3HeadObjectAttributes.java | 10 +-
.../ozone/client/io/OzoneCryptoInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 28 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 41 +-
.../hadoop/ozone/client/rpc/TestRpcClient.java | 17 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 73 +++-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 +
.../ozone/om/helpers/BucketDeletedBytes.java | 79 ++++
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 57 ++-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +-
.../om/protocolPB/OzoneManagerClientProtocol.java | 10 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 86 +++-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 10 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 114 +++++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 108 +++++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 26 +-
hadoop-ozone/dev-support/checks/junit.sh | 3 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 16 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 16 +-
.../src/main/smoketest/s3/secretgenerate.robot | 3 +-
hadoop-ozone/httpfsgateway/pom.xml | 12 +-
.../fs/http/server/HttpFSServerWebServer.java | 4 +
.../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +-
.../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +-
hadoop-ozone/iceberg/pom.xml | 24 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +-
hadoop-ozone/integration-test-s3/pom.xml | 40 +-
.../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +-
.../org/apache/hadoop/ozone/s3/ProxyServer.java | 22 +-
.../ozone/s3/ProxyServerIntegrationTest.java | 4 +-
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 115 +++++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 453 ++++++++++++++++++++-
hadoop-ozone/integration-test/pom.xml | 50 +--
.../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++++++++++++++++
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 ++
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +-
.../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +-
.../client/rpc/TestBlockDataStreamOutput.java | 42 +-
.../rpc/TestContainerStateMachineStream.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 127 ++++--
.../rpc/read/TestStreamBlockInputStream.java | 88 ++--
.../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +-
.../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +++-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 8 +-
.../TestOmMixedWorkloadUnderDeletionBench.java | 19 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 125 +++---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 56 ++-
.../src/main/proto/OmClientProtocol.proto | 35 +-
hadoop-ozone/ozone-manager/pom.xml | 12 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 6 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 226 ++++++++--
.../apache/hadoop/ozone/om/S3SecretManager.java | 4 +
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 37 ++
.../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +-
.../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 +++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +-
.../hadoop/ozone/om/ratis/OMThreadContext.java | 198 +++++++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 26 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 53 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 26 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 30 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/response/CleanupTableInfo.java | 51 ---
.../ozone/om/response/DummyOMClientResponse.java | 1 -
.../hadoop/ozone/om/response/OMClientResponse.java | 24 +-
.../om/response/bucket/OMBucketCreateResponse.java | 5 -
.../om/response/bucket/OMBucketDeleteResponse.java | 6 -
.../response/bucket/OMBucketSetOwnerResponse.java | 4 -
.../bucket/OMBucketSetPropertyResponse.java | 4 -
.../response/bucket/acl/OMBucketAclResponse.java | 4 -
.../response/file/OMDirectoryCreateResponse.java | 4 -
.../file/OMDirectoryCreateResponseWithFSO.java | 14 +-
.../om/response/file/OMFileCreateResponse.java | 5 -
.../response/file/OMFileCreateResponseWithFSO.java | 7 -
.../om/response/file/OMRecoverLeaseResponse.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 5 -
.../key/OMAllocateBlockResponseWithFSO.java | 5 -
.../key/OMDirectoriesPurgeResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 8 -
.../response/key/OMKeyCommitResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 -
.../response/key/OMKeyCreateResponseWithFSO.java | 7 -
.../ozone/om/response/key/OMKeyDeleteResponse.java | 7 -
.../response/key/OMKeyDeleteResponseWithFSO.java | 10 -
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 -
.../ozone/om/response/key/OMKeyRenameResponse.java | 5 -
.../response/key/OMKeyRenameResponseWithFSO.java | 7 -
.../om/response/key/OMKeySetTimesResponse.java | 4 -
.../response/key/OMKeySetTimesResponseWithFSO.java | 5 -
.../om/response/key/OMKeysDeleteResponse.java | 7 -
.../response/key/OMKeysDeleteResponseWithFSO.java | 11 -
.../om/response/key/OMKeysRenameResponse.java | 4 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 8 -
.../om/response/key/acl/OMKeyAclResponse.java | 4 -
.../response/key/acl/OMKeyAclResponseWithFSO.java | 5 -
.../key/acl/prefix/OMPrefixAclResponse.java | 4 -
.../OMLifecycleConfigurationDeleteResponse.java | 4 -
.../OMLifecycleConfigurationSetResponse.java | 4 -
.../OMLifecycleSaveScanStateResponse.java | 3 -
.../OMLifecycleSetServiceStatusResponse.java | 2 -
.../S3ExpiredMultipartUploadsAbortResponse.java | 10 -
.../S3InitiateMultipartUploadResponse.java | 5 -
.../S3InitiateMultipartUploadResponseWithFSO.java | 7 -
.../multipart/S3MultipartUploadAbortResponse.java | 9 -
.../S3MultipartUploadAbortResponseWithFSO.java | 9 -
.../S3MultipartUploadCommitPartResponse.java | 8 -
...S3MultipartUploadCommitPartResponseWithFSO.java | 9 -
.../S3MultipartUploadCompleteResponse.java | 10 -
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 -
.../response/s3/security/OMSetSecretResponse.java | 4 -
.../response/s3/security/S3AssumeRoleResponse.java | 2 -
.../security/S3DeleteRevokedSTSTokensResponse.java | 3 -
.../response/s3/security/S3GetSecretResponse.java | 4 -
.../s3/security/S3RevokeSTSTokenResponse.java | 3 -
.../s3/security/S3RevokeSecretResponse.java | 4 -
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 -
.../S3DeleteObjectTaggingResponseWithFSO.java | 4 -
.../s3/tagging/S3PutObjectTaggingResponse.java | 4 -
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 -
.../tenant/OMSetRangerServiceVersionResponse.java | 4 -
.../s3/tenant/OMTenantAssignAdminResponse.java | 6 -
.../tenant/OMTenantAssignUserAccessIdResponse.java | 10 -
.../response/s3/tenant/OMTenantCreateResponse.java | 8 -
.../response/s3/tenant/OMTenantDeleteResponse.java | 8 -
.../s3/tenant/OMTenantRevokeAdminResponse.java | 6 -
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 -
.../security/OMCancelDelegationTokenResponse.java | 4 -
.../security/OMGetDelegationTokenResponse.java | 4 -
.../security/OMRenewDelegationTokenResponse.java | 4 -
.../snapshot/OMSnapshotCreateResponse.java | 7 -
.../snapshot/OMSnapshotDeleteResponse.java | 4 -
.../OMSnapshotMoveDeletedKeysResponse.java | 3 -
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 -
.../response/snapshot/OMSnapshotPurgeResponse.java | 8 -
.../snapshot/OMSnapshotRenameResponse.java | 4 -
.../snapshot/OMSnapshotSetPropertyResponse.java | 4 -
.../upgrade/OMCompleteFinalizeUpgradeResponse.java | 3 -
.../upgrade/OMStartFinalizeUpgradeResponse.java | 4 -
.../om/response/util/OMEchoRPCWriteResponse.java | 2 -
.../om/response/volume/OMQuotaRepairResponse.java | 5 -
.../om/response/volume/OMVolumeAclOpResponse.java | 4 -
.../om/response/volume/OMVolumeCreateResponse.java | 4 -
.../om/response/volume/OMVolumeDeleteResponse.java | 4 -
.../response/volume/OMVolumeSetOwnerResponse.java | 4 -
.../response/volume/OMVolumeSetQuotaResponse.java | 4 -
.../ozone/om/service/KeyLifecycleService.java | 10 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 129 +++++-
.../ozone/om/snapshot/SnapshotDiffManager.java | 15 +-
.../trapped/BucketDeletedDataCalculator.java | 373 +++++++++++++++++
.../ozone/om/snapshot/trapped}/package-info.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 46 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 27 +-
.../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 ++++
.../ozone/om/TestOmSnapshotLeakDetection.java | 87 ++++
.../org/apache/hadoop/ozone/om/TestScmClient.java | 24 ++
.../ozone/om/execution/TestOMExecutionFlow.java | 165 ++++++++
.../ozone/om/lock/TestOMLockDetailsUtil.java | 78 ++++
.../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++++++++++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +++-
.../om/ratis/TestOzoneManagerStateMachine.java | 226 ++++++++++
.../om/request/key/TestOMKeyCreateRequest.java | 9 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 7 +
.../ozone/om/response/TestCleanupTableInfo.java | 384 -----------------
.../ozone/om/response/TestOMClientResponse.java} | 42 +-
.../hadoop/ozone/om/response/TestOMResponse.java | 51 ++-
.../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 +++++++++++
.../file/TestOMDirectoryCreateResponseWithFSO.java | 2 -
.../ozone/om/service/TestKeyLifecycleService.java | 10 +-
.../ozone/om/service/TestQuotaRepairTask.java | 425 +++++++++++++++++++
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 +
.../trapped/TestBucketDeletedDataCalculator.java | 243 +++++++++++
hadoop-ozone/recon/pom.xml | 12 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/filters/ReconAuthFilter.java | 2 +-
.../webapps/recon/ozone-recon-web/package.json | 4 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +-
.../src/v2/components/navBar/navBar.tsx | 12 +-
.../src/v2/pages/heatmap/heatmap.tsx | 35 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 6 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 3 +-
.../ReadConsistencyFilter.java} | 40 +-
.../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 ++++++
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 +
.../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 9 +-
.../apache/hadoop/ozone/s3/S3STSHttpServer.java | 3 +
.../hadoop/ozone/s3/SignedChunksInputStream.java | 152 ++++++-
.../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 +++
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 +
.../s3/endpoint/BucketGetLocationHandler.java | 5 +
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 9 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 21 +-
.../BucketOperationNotImplementedHandler.java | 64 +++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 59 ++-
.../s3/endpoint/ListMultipartUploadsHandler.java | 15 +-
.../ozone/s3/endpoint/MultipartKeyHandler.java | 12 +-
.../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +-
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 16 +-
.../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 +
.../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +-
.../ObjectOperationNotImplementedHandler.java | 65 +++
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 8 +-
.../hadoop/ozone/s3/endpoint/S3RequestContext.java | 12 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../ozone/s3/signature/AWSSignatureProcessor.java | 3 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 43 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../hadoop/ozone/s3/signature/ChunksValidator.java | 39 +-
.../hadoop/ozone/s3/signature/Credential.java | 20 +
.../s3/signature/MalformedResourceException.java | 19 +-
.../ozone/s3/signature/StringToSignProducer.java | 50 ---
.../ozone/s3/util/ReadConsistencyContext.java | 112 +++++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 45 ++
.../hadoop/ozone/s3/util/S3GActionIamMapper.java | 2 +
.../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 +
.../hadoop/ozone/client/ClientProtocolStub.java | 42 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 94 +++--
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 +++
.../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 +++
.../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 ++++++
.../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 +++++
.../ozone/s3/TestSignedChunksInputStream.java | 173 ++++++++
.../ozone/s3/endpoint/EndpointTestUtils.java | 20 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++++++++----------
.../s3/endpoint/TestBucketNotImplemented.java | 102 +++++
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 189 +++++++++
.../s3/endpoint/TestListMultipartUploads.java | 166 ++++++++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +++++
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 146 ++++++-
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 24 ++
.../s3/endpoint/TestObjectNotImplemented.java | 113 +++++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 130 +++++-
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +-
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 39 ++
.../ozone/s3/endpoint/TestPermissionCheck.java | 10 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 111 ++++-
.../TestS3LifecycleConfigurationDelete.java | 125 ++++++
.../endpoint/TestS3LifecycleConfigurationGet.java | 90 ++++
.../endpoint/TestS3LifecycleConfigurationPut.java | 90 ++++
.../ozone/s3/endpoint/TestS3RequestContext.java | 86 ++++
.../ozone/s3/endpoint/TestUploadWithStream.java | 34 ++
.../ozone/s3/signature/SignatureTestUtils.java | 34 ++
.../signature/TestAuthorizationV4HeaderParser.java | 133 +++---
.../signature/TestAuthorizationV4QueryParser.java | 55 ++-
.../ozone/s3/signature/TestChunksValidator.java | 56 +++
.../s3/signature/TestStringToSignProducer.java | 56 +--
.../ozone/s3/util/TestS3GActionIamMapper.java | 2 +
.../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +-
.../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 +
pom.xml | 77 ++--
350 files changed, 12741 insertions(+), 2677 deletions(-)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/TestGrpcXceiverService.java
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/{AdminAuthorizedServlet.java => HddsJMXJsonServlet.java} (52%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCacheUpdateTracker.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/proxy/TestSCMFailoverProxyProviderChangeConfig.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTableCacheUpdateTracker.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHADBTransactionBufferImpl.java
create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestOmSCMNodesReconfiguration.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockDetailsUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMThreadContext.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped}/package-info.java (85%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestEDEKCacheLoader.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLeakDetection.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/execution/TestOMExecutionFlow.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOMLockDetailsUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOMThreadContext.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
copy hadoop-ozone/ozone-manager/src/{main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java => test/java/org/apache/hadoop/ozone/om/response/TestOMClientResponse.java} (52%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestS3DerivedKey.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/{s3sts/S3STSEnabledEndpointRequestFilter.java => s3/ReadConsistencyFilter.java} (51%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3ContentTypeFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/WeldCdiIntegration.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ReadConsistencyContext.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestReadConsistencyFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3ContentTypeFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHttpServers.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketNotImplemented.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListMultipartUploads.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3RequestContext.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge branch 'master' into HDDS-14496-zdu
Posted by er...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a9a64dbed31ba1dfee9057b7c8ddd42acb948b2f
Merge: c064bafb5e1 a99fabb9dea
Author: Ethan Rose <er...@apache.org>
AuthorDate: Tue Oct 6 13:56:01 2026 -0400
Merge branch 'master' into HDDS-14496-zdu
* master: (64 commits)
HDDS-16716. Add description for ozone.scm.ec.pipeline.per.volume.factor (#11415)
HDDS-16008. PutBlocks from Flushes also go without Raft (#11356)
HDDS-16666. Flush SCM transaction in memory during apply transaction (#11409)
HDDS-15749. Run specific JUnit tests if possible (#10671)
HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for FSO buckets (#11242).
HDDS-16643. Remove CleanupTableInfo mechanism (#11365)
HDDS-16721. StreamBlockInputStream.read() returns a negative value for bytes 0x80 to 0xFF (#11411)
HDDS-16241. gRPC deadline kills long-lived block streams after 30 seconds and the client never recovers (#11080)
HDDS-15991. Speed up deleted table scans in quota repair (#11386)
HDDS-16674. Bump awssdk to 2.55.6 (#11407)
HDDS-16673. Avoid redundant ListBuckets RPCs when S3 bucket listing reaches the end (#11397)
HDDS-16708. Let dependabot ignore iceberg minor version upgrades (#11398)
HDDS-16713. Bump develocity-maven-extension to 2.6.0 (#11405)
HDDS-16300. Allow OM to dynamically reconfigure its SCM node list without a restart (#11218)
HDDS-15089. Support S3 per request read consistency (#11252)
HDDS-16704. ReadBlock fails with IllegalStateException when a response is shorter than responseDataSize (#11402)
HDDS-16631. Fix chooseRandom for rack names with common prefixes (#11401)
HDDS-16654. Replace usage of deprecated finalize() in OM (#11376)
HDDS-16658. Reuse source key details when opening input stream in S3 CopyObject (#11396)
HDDS-16711. Bump moment to 2.31.0 (#11373)
...
Conflicts:
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCancelPrepareResponse.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMPrepareResponse.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
.github/dependabot.yml | 8 +-
.github/workflows/build-ratis.yml | 2 +-
.github/workflows/check.yml | 2 +-
.github/workflows/ci.yml | 35 +-
.github/workflows/intermittent-test-check.yml | 2 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
dev-support/ci/selective_ci_checks.bats | 42 +-
dev-support/ci/selective_ci_checks.sh | 68 +++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 5 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 77 +++-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hdds/scm/storage/MultipartInputStream.java | 48 +--
.../hadoop/hdds/scm/storage/PartInputStream.java | 13 +
.../hdds/scm/storage/StreamBlockInputStream.java | 182 +++++++--
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +-
.../hdds/scm/storage/MockDatanodePipeline.java | 45 +-
.../scm/storage/TestBlockDataStreamOutput.java | 59 ++-
.../hdds/scm/storage/TestMultipartInputStream.java | 69 ++++
.../scm/storage/TestStreamBlockInputStream.java | 361 ++++++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 23 +-
.../hdds/utils/TestConnectionFailureUtils.java | 14 +
.../common/statemachine/DatanodeConfiguration.java | 22 +
.../ClosePipelineCommandHandler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 68 +++-
.../common/transport/server/XceiverServerGrpc.java | 5 +-
.../server/ratis/ContainerStateMachine.java | 31 +-
.../transport/server/ratis/DispatcherContext.java | 4 +-
.../common/transport/server/ratis/LocalStream.java | 22 +-
.../container/keyvalue/ReadBlockComputation.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 3 +-
.../transport/server/TestGrpcXceiverService.java | 169 ++++++++
.../container/keyvalue/TestKeyValueHandler.java | 18 +
.../docs/content/feature/Reconfigurability.md | 2 +
hadoop-hdds/framework/pom.xml | 22 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 91 ++++-
.../hdds/server/http/AdminAuthorizedServlet.java | 2 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 56 ++-
...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 244 ++++++++---
.../hdds/server/http/ServletElementsFactory.java | 4 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +
.../utils/db/cache/TableCacheUpdateTracker.java | 88 ++++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 19 +
.../TestSCMFailoverProxyProviderChangeConfig.java | 149 +++++++
.../hdds/server/http/TestBaseHttpServer.java | 53 +++
.../hadoop/hdds/server/http/TestHttpServer2.java | 390 ++++++++++++++++++
.../hdds/server/http/TestHttpServer2SSL.java | 36 +-
.../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 ++++++
hadoop-hdds/rocks-native/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 12 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 2 +
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 41 ++
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 5 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 --
.../hadoop/hdds/scm/node/NodeStateManager.java | 32 +-
.../hdds/scm/node/PendingContainerTracker.java | 2 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 221 ++++++----
.../scm/node/UnhealthyToHealthyNodeHandler.java | 15 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 5 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++++++++++++++
.../ha/TestSCMHATransactionBufferMonitorTask.java | 1 +
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 124 +++++-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 53 +++
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 57 +--
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 88 +++-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 63 ++-
.../admin/om/BucketDeletedBytesSubCommand.java | 101 +++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../ozone/client/S3HeadObjectAttributes.java | 10 +-
.../ozone/client/io/OzoneCryptoInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 28 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 41 +-
.../hadoop/ozone/client/rpc/TestRpcClient.java | 17 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 73 +++-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 +
.../ozone/om/helpers/BucketDeletedBytes.java | 79 ++++
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 57 ++-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +-
.../om/protocolPB/OzoneManagerClientProtocol.java | 10 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 86 +++-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 10 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 114 +++++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 108 +++++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 26 +-
hadoop-ozone/dev-support/checks/junit.sh | 3 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 16 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 16 +-
.../src/main/smoketest/s3/secretgenerate.robot | 3 +-
hadoop-ozone/httpfsgateway/pom.xml | 12 +-
.../fs/http/server/HttpFSServerWebServer.java | 4 +
.../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +-
.../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +-
hadoop-ozone/iceberg/pom.xml | 24 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +-
hadoop-ozone/integration-test-s3/pom.xml | 40 +-
.../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +-
.../org/apache/hadoop/ozone/s3/ProxyServer.java | 22 +-
.../ozone/s3/ProxyServerIntegrationTest.java | 4 +-
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 115 +++++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 453 ++++++++++++++++++++-
hadoop-ozone/integration-test/pom.xml | 50 +--
.../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++++++++++++++++
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 ++
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +-
.../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +-
.../client/rpc/TestBlockDataStreamOutput.java | 42 +-
.../rpc/TestContainerStateMachineStream.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 127 ++++--
.../rpc/read/TestStreamBlockInputStream.java | 88 ++--
.../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +-
.../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +++-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 8 +-
.../TestOmMixedWorkloadUnderDeletionBench.java | 19 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 125 +++---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 56 ++-
.../src/main/proto/OmClientProtocol.proto | 35 +-
hadoop-ozone/ozone-manager/pom.xml | 12 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 6 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 226 ++++++++--
.../apache/hadoop/ozone/om/S3SecretManager.java | 4 +
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 37 ++
.../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +-
.../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 +++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +-
.../hadoop/ozone/om/ratis/OMThreadContext.java | 198 +++++++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 26 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 53 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 26 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 30 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/response/CleanupTableInfo.java | 51 ---
.../ozone/om/response/DummyOMClientResponse.java | 1 -
.../hadoop/ozone/om/response/OMClientResponse.java | 24 +-
.../om/response/bucket/OMBucketCreateResponse.java | 5 -
.../om/response/bucket/OMBucketDeleteResponse.java | 6 -
.../response/bucket/OMBucketSetOwnerResponse.java | 4 -
.../bucket/OMBucketSetPropertyResponse.java | 4 -
.../response/bucket/acl/OMBucketAclResponse.java | 4 -
.../response/file/OMDirectoryCreateResponse.java | 4 -
.../file/OMDirectoryCreateResponseWithFSO.java | 14 +-
.../om/response/file/OMFileCreateResponse.java | 5 -
.../response/file/OMFileCreateResponseWithFSO.java | 7 -
.../om/response/file/OMRecoverLeaseResponse.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 5 -
.../key/OMAllocateBlockResponseWithFSO.java | 5 -
.../key/OMDirectoriesPurgeResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 8 -
.../response/key/OMKeyCommitResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 -
.../response/key/OMKeyCreateResponseWithFSO.java | 7 -
.../ozone/om/response/key/OMKeyDeleteResponse.java | 7 -
.../response/key/OMKeyDeleteResponseWithFSO.java | 10 -
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 -
.../ozone/om/response/key/OMKeyRenameResponse.java | 5 -
.../response/key/OMKeyRenameResponseWithFSO.java | 7 -
.../om/response/key/OMKeySetTimesResponse.java | 4 -
.../response/key/OMKeySetTimesResponseWithFSO.java | 5 -
.../om/response/key/OMKeysDeleteResponse.java | 7 -
.../response/key/OMKeysDeleteResponseWithFSO.java | 11 -
.../om/response/key/OMKeysRenameResponse.java | 4 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 8 -
.../om/response/key/acl/OMKeyAclResponse.java | 4 -
.../response/key/acl/OMKeyAclResponseWithFSO.java | 5 -
.../key/acl/prefix/OMPrefixAclResponse.java | 4 -
.../OMLifecycleConfigurationDeleteResponse.java | 4 -
.../OMLifecycleConfigurationSetResponse.java | 4 -
.../OMLifecycleSaveScanStateResponse.java | 3 -
.../OMLifecycleSetServiceStatusResponse.java | 2 -
.../S3ExpiredMultipartUploadsAbortResponse.java | 10 -
.../S3InitiateMultipartUploadResponse.java | 5 -
.../S3InitiateMultipartUploadResponseWithFSO.java | 7 -
.../multipart/S3MultipartUploadAbortResponse.java | 9 -
.../S3MultipartUploadAbortResponseWithFSO.java | 9 -
.../S3MultipartUploadCommitPartResponse.java | 8 -
...S3MultipartUploadCommitPartResponseWithFSO.java | 9 -
.../S3MultipartUploadCompleteResponse.java | 10 -
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 -
.../response/s3/security/OMSetSecretResponse.java | 4 -
.../response/s3/security/S3AssumeRoleResponse.java | 2 -
.../security/S3DeleteRevokedSTSTokensResponse.java | 3 -
.../response/s3/security/S3GetSecretResponse.java | 4 -
.../s3/security/S3RevokeSTSTokenResponse.java | 3 -
.../s3/security/S3RevokeSecretResponse.java | 4 -
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 -
.../S3DeleteObjectTaggingResponseWithFSO.java | 4 -
.../s3/tagging/S3PutObjectTaggingResponse.java | 4 -
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 -
.../tenant/OMSetRangerServiceVersionResponse.java | 4 -
.../s3/tenant/OMTenantAssignAdminResponse.java | 6 -
.../tenant/OMTenantAssignUserAccessIdResponse.java | 10 -
.../response/s3/tenant/OMTenantCreateResponse.java | 8 -
.../response/s3/tenant/OMTenantDeleteResponse.java | 8 -
.../s3/tenant/OMTenantRevokeAdminResponse.java | 6 -
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 -
.../security/OMCancelDelegationTokenResponse.java | 4 -
.../security/OMGetDelegationTokenResponse.java | 4 -
.../security/OMRenewDelegationTokenResponse.java | 4 -
.../snapshot/OMSnapshotCreateResponse.java | 7 -
.../snapshot/OMSnapshotDeleteResponse.java | 4 -
.../OMSnapshotMoveDeletedKeysResponse.java | 3 -
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 -
.../response/snapshot/OMSnapshotPurgeResponse.java | 8 -
.../snapshot/OMSnapshotRenameResponse.java | 4 -
.../snapshot/OMSnapshotSetPropertyResponse.java | 4 -
.../upgrade/OMCompleteFinalizeUpgradeResponse.java | 3 -
.../upgrade/OMStartFinalizeUpgradeResponse.java | 4 -
.../om/response/util/OMEchoRPCWriteResponse.java | 2 -
.../om/response/volume/OMQuotaRepairResponse.java | 5 -
.../om/response/volume/OMVolumeAclOpResponse.java | 4 -
.../om/response/volume/OMVolumeCreateResponse.java | 4 -
.../om/response/volume/OMVolumeDeleteResponse.java | 4 -
.../response/volume/OMVolumeSetOwnerResponse.java | 4 -
.../response/volume/OMVolumeSetQuotaResponse.java | 4 -
.../ozone/om/service/KeyLifecycleService.java | 10 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 129 +++++-
.../ozone/om/snapshot/SnapshotDiffManager.java | 15 +-
.../trapped/BucketDeletedDataCalculator.java | 373 +++++++++++++++++
.../ozone/om/snapshot/trapped/package-info.java | 19 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 46 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 27 +-
.../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 ++++
.../ozone/om/TestOmSnapshotLeakDetection.java | 87 ++++
.../org/apache/hadoop/ozone/om/TestScmClient.java | 24 ++
.../ozone/om/execution/TestOMExecutionFlow.java | 165 ++++++++
.../ozone/om/lock/TestOMLockDetailsUtil.java | 78 ++++
.../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++++++++++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +++-
.../om/ratis/TestOzoneManagerStateMachine.java | 226 ++++++++++
.../om/request/key/TestOMKeyCreateRequest.java | 9 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 7 +
.../ozone/om/response/TestCleanupTableInfo.java | 384 -----------------
.../ozone/om/response/TestOMClientResponse.java | 51 +++
.../hadoop/ozone/om/response/TestOMResponse.java | 51 ++-
.../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 +++++++++++
.../file/TestOMDirectoryCreateResponseWithFSO.java | 2 -
.../ozone/om/service/TestKeyLifecycleService.java | 10 +-
.../ozone/om/service/TestQuotaRepairTask.java | 425 +++++++++++++++++++
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 +
.../trapped/TestBucketDeletedDataCalculator.java | 243 +++++++++++
hadoop-ozone/recon/pom.xml | 12 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/filters/ReconAuthFilter.java | 2 +-
.../webapps/recon/ozone-recon-web/package.json | 4 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +-
.../src/v2/components/navBar/navBar.tsx | 12 +-
.../src/v2/pages/heatmap/heatmap.tsx | 35 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 6 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 3 +-
.../hadoop/ozone/s3/ReadConsistencyFilter.java | 53 +++
.../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 ++++++
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 +
.../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 9 +-
.../apache/hadoop/ozone/s3/S3STSHttpServer.java | 3 +
.../hadoop/ozone/s3/SignedChunksInputStream.java | 152 ++++++-
.../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 +++
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 +
.../s3/endpoint/BucketGetLocationHandler.java | 5 +
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 9 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 21 +-
.../BucketOperationNotImplementedHandler.java | 64 +++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 59 ++-
.../s3/endpoint/ListMultipartUploadsHandler.java | 15 +-
.../ozone/s3/endpoint/MultipartKeyHandler.java | 12 +-
.../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +-
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 16 +-
.../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 +
.../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +-
.../ObjectOperationNotImplementedHandler.java | 65 +++
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 8 +-
.../hadoop/ozone/s3/endpoint/S3RequestContext.java | 12 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../ozone/s3/signature/AWSSignatureProcessor.java | 3 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 43 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../hadoop/ozone/s3/signature/ChunksValidator.java | 39 +-
.../hadoop/ozone/s3/signature/Credential.java | 20 +
.../s3/signature/MalformedResourceException.java | 19 +-
.../ozone/s3/signature/StringToSignProducer.java | 50 ---
.../ozone/s3/util/ReadConsistencyContext.java | 112 +++++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 45 ++
.../hadoop/ozone/s3/util/S3GActionIamMapper.java | 2 +
.../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 +
.../hadoop/ozone/client/ClientProtocolStub.java | 42 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 94 +++--
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 +++
.../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 +++
.../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 ++++++
.../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 +++++
.../ozone/s3/TestSignedChunksInputStream.java | 173 ++++++++
.../ozone/s3/endpoint/EndpointTestUtils.java | 20 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++++++++----------
.../s3/endpoint/TestBucketNotImplemented.java | 102 +++++
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 189 +++++++++
.../s3/endpoint/TestListMultipartUploads.java | 166 ++++++++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +++++
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 146 ++++++-
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 24 ++
.../s3/endpoint/TestObjectNotImplemented.java | 113 +++++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 130 +++++-
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +-
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 39 ++
.../ozone/s3/endpoint/TestPermissionCheck.java | 10 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 111 ++++-
.../TestS3LifecycleConfigurationDelete.java | 125 ++++++
.../endpoint/TestS3LifecycleConfigurationGet.java | 90 ++++
.../endpoint/TestS3LifecycleConfigurationPut.java | 90 ++++
.../ozone/s3/endpoint/TestS3RequestContext.java | 86 ++++
.../ozone/s3/endpoint/TestUploadWithStream.java | 34 ++
.../ozone/s3/signature/SignatureTestUtils.java | 34 ++
.../signature/TestAuthorizationV4HeaderParser.java | 133 +++---
.../signature/TestAuthorizationV4QueryParser.java | 55 ++-
.../ozone/s3/signature/TestChunksValidator.java | 56 +++
.../s3/signature/TestStringToSignProducer.java | 56 +--
.../ozone/s3/util/TestS3GActionIamMapper.java | 2 +
.../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +-
.../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 +
pom.xml | 77 ++--
350 files changed, 12802 insertions(+), 2653 deletions(-)
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index cd5d01d05a3,51df2cf7879..0563f9d6eea
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@@ -148,10 -166,15 +153,12 @@@ public class NodeStateManager implement
*
* @param conf Configuration
* @param eventPublisher event publisher
+ * @param clusterMap network topology of the cluster
- * @param layoutManager Layout version manager
*/
public NodeStateManager(ConfigurationSource conf,
EventPublisher eventPublisher,
+ NetworkTopology clusterMap,
- LayoutVersionManager layoutManager,
SCMContext scmContext) {
- this.layoutVersionManager = layoutManager;
this.nodeStateMap = new NodeStateMap();
this.node2PipelineMap = new Node2PipelineMap();
this.eventPublisher = eventPublisher;
@@@ -237,13 -314,13 +245,13 @@@
* @throws NodeAlreadyExistsException if the node is already present
*/
public void addNode(DatanodeDetails datanodeDetails,
- LayoutVersionProto layoutInfo) throws NodeAlreadyExistsException {
- nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, layoutInfo));
+ DatanodeVersionProto versionInfo) throws NodeAlreadyExistsException {
+ nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, versionInfo));
try {
- updateLastKnownLayoutVersion(datanodeDetails, layoutInfo);
+ updateLastKnownVersionInfo(datanodeDetails, versionInfo);
} catch (NodeNotFoundException ex) {
throw new IllegalStateException("Inconsistent NodeStateMap! Datanode "
- + datanodeDetails.getID() + " was added but not found in map: " + nodeStateMap);
+ + datanodeDetails + " was added but not found in map: " + nodeStateMap);
}
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index cd6a0e1fdf1,d8884cdc464..3b640d6e8cc
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@@ -193,10 -192,11 +193,10 @@@ public class SCMNodeManager implements
EventPublisher eventPublisher,
NetworkTopology networkTopology,
SCMContext scmContext,
- HDDSLayoutVersionManager layoutVersionManager,
+ ScmVersionManager versionManager,
Function<String, String> nodeResolver) {
this.scmNodeEventPublisher = eventPublisher;
- this.nodeStateManager = new NodeStateManager(conf, eventPublisher, scmContext);
- this.nodeStateManager = new NodeStateManager(conf, eventPublisher, networkTopology,
- layoutVersionManager, scmContext);
++ this.nodeStateManager = new NodeStateManager(conf, eventPublisher, networkTopology, scmContext);
this.version = VersionInfo.getLatestVersion();
this.commandQueue = new CommandQueue();
this.scmStorageConfig = scmStorageConfig;
@@@ -473,10 -476,7 +473,7 @@@
if (updateDnsToDnIdMap(oldNode.getHostName(), oldNode.getIpAddress(),
hostName, ipAddress, dnId)) {
LOG.info("Updating datanode from {} to {}", oldNode, datanodeDetails);
- clusterMap.update(oldNode, datanodeDetails);
- nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
- DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails);
- Preconditions.checkState(dn.getParent() != null);
- DatanodeDetails dn = updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo);
++ DatanodeDetails dn = updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo);
processNodeReport(datanodeDetails, nodeReport);
LOG.info("Updated datanode to: {}", dn);
scmNodeEventPublisher.fireEvent(SCMEvents.NODE_ADDRESS_UPDATE, dn);
@@@ -484,12 -484,12 +481,12 @@@
LOG.info("Update the version for registered datanode {}, " +
"oldVersion = {}, newVersion = {}.",
datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion());
- nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
- updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo);
++ updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo);
} else if (oldNode.portsChanged(datanodeDetails)) {
// Refresh the stored node when its port set changes
LOG.info("Updating ports for registered datanode {}: {} -> {}",
datanodeDetails, oldNode.getPorts(), datanodeDetails.getPorts());
- nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
- updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo);
++ updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo);
}
} catch (NodeNotFoundException e) {
LOG.error("Cannot find datanode {} from nodeStateManager",
@@@ -503,6 -503,25 +500,25 @@@
.build();
}
+ /**
+ * Replaces the record of an already registered datanode, in both the node state and the network
+ * topology. The topology has to be updated first: {@link NetworkTopology#update} sets the parent on
+ * the new node, and {@link NodeStateManager#updateNode} only picks it up because the stored
+ * {@link DatanodeInfo} is copied from that node afterwards. Updating the node state alone would
+ * leave the stale record in the topology and the stored node without a parent.
+ *
+ * @return the refreshed node as held by the node state manager
+ */
+ private DatanodeDetails updateNodeAndTopology(DatanodeInfo oldNode,
- DatanodeDetails datanodeDetails, LayoutVersionProto layoutInfo)
++ DatanodeDetails datanodeDetails, DatanodeVersionProto dnVersionInfo)
+ throws NodeNotFoundException {
+ clusterMap.update(oldNode, datanodeDetails);
- nodeStateManager.updateNode(datanodeDetails, layoutInfo);
++ nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
+ final DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails);
+ Preconditions.checkState(dn.getParent() != null);
+ return dn;
+ }
+
/**
* Add an entry to the dnsToUuidMap, which maps hostname / IP to the DNs
* running on that host. As each address can have many DNs running on it,
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java
index 56bda449847,eba89247a19..2d015ee723d
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java
@@@ -17,42 -17,35 +17,29 @@@
package org.apache.hadoop.hdds.scm.node;
- import java.util.Objects;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.ha.SCMService.Event;
+import org.apache.hadoop.hdds.scm.ha.SCMService;
import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
- import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Handles Read Only healthy to healthy node event. (Possibly due to a
- * datanode having finalized)
+ * Handles non healthy to healthy node event.
*/
-public class ReadOnlyHealthyToHealthyNodeHandler
- implements EventHandler<DatanodeDetails> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ReadOnlyHealthyToHealthyNodeHandler.class);
+public class UnhealthyToHealthyNodeHandler implements EventHandler<DatanodeDetails> {
+ private static final Logger LOG = LoggerFactory.getLogger(UnhealthyToHealthyNodeHandler.class);
private final SCMServiceManager serviceManager;
- private final NodeManager nodeManager;
- public UnhealthyToHealthyNodeHandler(NodeManager nodeManager, SCMServiceManager serviceManager) {
- public ReadOnlyHealthyToHealthyNodeHandler(SCMServiceManager serviceManager) {
++ public UnhealthyToHealthyNodeHandler(SCMServiceManager serviceManager) {
this.serviceManager = serviceManager;
- this.nodeManager = nodeManager;
}
@Override
- public void onMessage(DatanodeDetails datanodeDetails,
- EventPublisher publisher) {
- LOG.info("Datanode {} moved to HEALTHY state.",
- datanodeDetails);
- serviceManager.notifyEventTriggered(
- Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED);
+ public void onMessage(DatanodeDetails datanodeDetails, EventPublisher publisher) {
+ LOG.info("Datanode {} moved to HEALTHY state.", datanodeDetails);
-
- //add node back if it is not present in networkTopology
- NetworkTopology nt = nodeManager.getClusterNetworkTopologyMap();
- if (!nt.contains(datanodeDetails)) {
- nt.add(datanodeDetails);
- // make sure after DN is added back into topology, DatanodeDetails
- // instance returned from nodeStateManager has parent correctly set.
- Objects.requireNonNull(nodeManager.getNode(datanodeDetails.getID()).getParent(), "Parent == null");
- }
+ serviceManager.notifyEventTriggered(SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED);
}
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 09e656b7a75,12665164c85..0fa0e38bfa7
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -507,8 -512,12 +507,8 @@@ public final class StorageContainerMana
pipelineManager, containerManager, null);
StartDatanodeAdminHandler datanodeStartAdminHandler =
new StartDatanodeAdminHandler(scmNodeManager, pipelineManager);
- ReadOnlyHealthyToHealthyNodeHandler readOnlyHealthyToHealthyNodeHandler =
- new ReadOnlyHealthyToHealthyNodeHandler(serviceManager);
- HealthyReadOnlyNodeHandler
- healthyReadOnlyNodeHandler =
- new HealthyReadOnlyNodeHandler(scmNodeManager,
- pipelineManager);
+ UnhealthyToHealthyNodeHandler unhealthyToHealthyNodeHandler =
- new UnhealthyToHealthyNodeHandler(scmNodeManager, serviceManager);
++ new UnhealthyToHealthyNodeHandler(serviceManager);
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
ContainerReportHandler containerReportHandler =
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
index 314c676d7c0,0f6703cdf57..0bf82bad2b5
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
@@@ -20,18 -20,26 +20,29 @@@ package org.apache.hadoop.hdds.scm.ha
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+ import static org.mockito.ArgumentMatchers.anyString;
+ import static org.mockito.Mockito.doThrow;
+ import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.times;
+ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+ import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
+ import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+ import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.ha.invoker.ScmInvoker;
+ import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
+ import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.utils.TransactionInfo;
+import org.apache.hadoop.ozone.upgrade.UpgradeException;
import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.LogEntryProto;
+import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto;
+ import org.apache.ratis.protocol.Message;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.util.ExitUtils;
@@@ -59,48 -67,101 +71,128 @@@ public class TestSCMStateMachine
metrics.unRegister();
}
-
+
+ /**
+ * A finalization step that throws an UpgradeException (an IOException, not an SCMException) must
+ * crash SCM rather than be returned to the Ratis client. UpgradeException skips the inner
+ * catch (SCMException) and hits the outer catch (Exception) -> ExitUtils.terminate.
+ */
+ @Test
+ public void testUpgradeExceptionDuringApplyTerminates() throws Exception {
+ ExitUtils.disableSystemExit();
++ try {
++ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
++ ScmInvoker<?> invoker = mock(ScmInvoker.class);
++ when(invoker.invokeLocal(any(), any())).thenThrow(
++ new UpgradeException(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED));
++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.FINALIZE, invoker);
++
++ // terminate throws ExitException when system exit is disabled
++ assertThrows(ExitUtils.ExitException.class,
++ () -> stateMachine.applyTransaction(newTransaction(RequestType.FINALIZE, 1)));
++ } finally {
++ ExitUtils.clear();
++ }
++ }
+
- StorageContainerManager scm = mock(StorageContainerManager.class);
- SCMMetrics metrics = SCMMetrics.create();
- when(scm.getMetrics()).thenReturn(metrics);
+ @Test
+ public void testApplyTransactionFlushesAfterRecordingTransactionInfo() throws Exception {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
- SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker());
++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker());
+
- CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(7));
++ CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 7));
+
+ assertTrue(result.isDone() && !result.isCompletedExceptionally());
+ InOrder order = inOrder(buffer);
+ order.verify(buffer).beginApplyingTransaction();
+ order.verify(buffer).updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(1, 7)));
+ order.verify(buffer).flushIfPendingLimitReached();
+ order.verify(buffer).endApplyingTransaction();
+ }
+ @Test
+ public void testFlushCheckedForEveryAppliedTransaction() throws Exception {
SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
- when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0)));
- SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker());
++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker());
- // The contents of the state machine are mocked, so SCMStateMachine#close is a no-op.
- SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer);
+ for (int i = 1; i <= 5; i++) {
- stateMachine.applyTransaction(newTransaction(i));
++ stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, i));
+ }
+
+ verify(buffer, times(5)).flushIfPendingLimitReached();
+ verify(buffer, times(5)).endApplyingTransaction();
+ }
-
++
+ @Test
+ public void testFlushStillCheckedWhenTransactionIsLogicallyRejected() throws Exception {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
ScmInvoker<?> invoker = mock(ScmInvoker.class);
- when(invoker.invokeLocal(any(), any())).thenThrow(
- new UpgradeException(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED));
- stateMachine.registerInvoker(RequestType.FINALIZE, invoker);
+ when(invoker.invokeLocal(anyString(), any()))
+ .thenThrow(new SCMException("rejected", ResultCodes.FAILED_TO_FIND_CONTAINER));
- SCMStateMachine stateMachine = newStateMachine(buffer, invoker);
++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, invoker);
+
- CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(3));
++ CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 3));
+
+ assertTrue(result.isCompletedExceptionally());
+ InOrder order = inOrder(buffer);
+ order.verify(buffer).updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(1, 3)));
+ order.verify(buffer).flushIfPendingLimitReached();
+ order.verify(buffer).endApplyingTransaction();
+ }
+
+ @Test
+ public void testFlushFailureTerminatesAndClosesApplyingWindow() throws Exception {
+ ExitUtils.disableSystemExit();
+ try {
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
+ doThrow(new IllegalStateException("injected flush failure")).when(buffer).flushIfPendingLimitReached();
- SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker());
++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker());
+
- assertThrows(ExitUtils.ExitException.class, () -> stateMachine.applyTransaction(newTransaction(9)));
++ assertThrows(ExitUtils.ExitException.class,
++ () -> stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 9)));
+
+ verify(buffer).endApplyingTransaction();
+ } finally {
+ ExitUtils.clear();
+ }
+ }
- SCMRatisRequest request = SCMRatisRequest.of(
- RequestType.FINALIZE, "finalize", new Class<?>[]{});
- private static TransactionContext newTransaction(long index) throws Exception {
- SCMRatisRequest request = SCMRatisRequest.of(RequestType.PIPELINE, "op", new Class<?>[0]);
- RaftProtos.StateMachineLogEntryProto smLogEntry = RaftProtos.StateMachineLogEntryProto.newBuilder()
++ private static TransactionContext newTransaction(RequestType type, long index) throws Exception {
++ // The operation name is only used by ScmInvoker#invokeLocal to pick a method to call. The invokers
++ // here are mocked and stubbed for any operation, so any non-empty name works.
++ SCMRatisRequest request = SCMRatisRequest.of(type, "op", new Class<?>[0]);
+ StateMachineLogEntryProto smLogEntry = StateMachineLogEntryProto.newBuilder()
.setLogData(request.encode().getContent())
.build();
- RaftProtos.LogEntryProto logEntry = RaftProtos.LogEntryProto.newBuilder()
+ LogEntryProto logEntry = LogEntryProto.newBuilder()
.setTerm(1)
- .setIndex(1)
+ .setIndex(index)
.setStateMachineLogEntry(smLogEntry)
.build();
TransactionContext trx = mock(TransactionContext.class);
when(trx.getStateMachineLogEntry()).thenReturn(smLogEntry);
when(trx.getLogEntry()).thenReturn(logEntry);
+ return trx;
+ }
- // terminate throws ExitException when system exit is disabled
- assertThrows(ExitUtils.ExitException.class,
- () -> stateMachine.applyTransaction(trx));
- private static SCMStateMachine newStateMachine(SCMHADBTransactionBuffer buffer, ScmInvoker<?> invoker) {
++ private static SCMStateMachine newStateMachine(SCMHADBTransactionBuffer buffer, RequestType type,
++ ScmInvoker<?> invoker) {
+ StorageContainerManager scm = mock(StorageContainerManager.class);
+ when(scm.getMetrics()).thenReturn(mock(SCMMetrics.class));
+ SCMContext context = mock(SCMContext.class);
+ when(context.isLeader()).thenReturn(true);
+ when(scm.getScmContext()).thenReturn(context);
+ when(scm.getDatanodeProtocolServer()).thenReturn(mock(SCMDatanodeProtocolServer.class));
+ when(scm.getScmSafeModeManager()).thenReturn(mock(SCMSafeModeManager.class));
+ when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0)));
+ SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer);
- stateMachine.registerInvoker(RequestType.PIPELINE, invoker);
++ stateMachine.registerInvoker(type, invoker);
+ return stateMachine;
+ }
- metrics.unRegister();
+ private static ScmInvoker<?> succeedingInvoker() throws Exception {
+ ScmInvoker<?> invoker = mock(ScmInvoker.class);
+ when(invoker.invokeLocal(anyString(), any())).thenReturn(Message.EMPTY);
+ return invoker;
}
}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 03e220d6d62,af006c7a41e..a50c6fa7caa
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@@ -293,12 -277,39 +277,39 @@@ public class TestDeadNodeHandler
.getContainerReplicas(ContainerID.valueOf(container3.getContainerID()));
assertEquals(1, container3Replicas.size());
assertEquals(datanode3, container3Replicas.iterator().next().getDatanodeDetails());
+ }
- //datanode will be added back to ClusterNetworkTopology if it resurrects
- unhealthyToHealthyNodeHandler.onMessage(datanode1, publisher);
- assertTrue(
- nodeManager.getClusterNetworkTopologyMap().contains(datanode1));
-
+ /**
+ * Verifies that DeadNodeHandler skips the event when the node is no longer DEAD, for example because it
- * heartbeated again and became HEALTHY_READONLY before the event was handled.
++ * heartbeated again and became HEALTHY before the event was handled.
+ */
+ @Test
+ public void testDeadNodeHandlerSkipsNodeThatIsNoLongerDead(
+ @TempDir File tempDir) throws Exception {
+ DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails();
+ String storagePath = tempDir.getPath()
+ .concat("/data-" + datanode.getID());
+ String metaStoragePath = tempDir.getPath()
+ .concat("/metadata-" + datanode.getID());
+ StorageReportProto storageReport = HddsTestUtils.createStorageReport(
+ datanode.getID(), storagePath, 100 * OzoneConsts.TB,
+ 10 * OzoneConsts.TB, 90 * OzoneConsts.TB, null);
+ MetadataStorageReportProto metaStorageReport =
+ HddsTestUtils.createMetadataStorageReport(metaStoragePath,
+ 100 * OzoneConsts.GB, 10 * OzoneConsts.GB,
+ 90 * OzoneConsts.GB, null);
+ nodeManager.register(datanode,
+ HddsTestUtils.createNodeReport(Arrays.asList(storageReport),
+ Arrays.asList(metaStorageReport)), null);
+ datanode = nodeManager.getNode(datanode.getID());
+
- // The DEAD_NODE event was fired, but the node heartbeated again and became HEALTHY_READONLY before
++ // The DEAD_NODE event was fired, but the node heartbeated again and became HEALTHY before
+ // DeadNodeHandler processed it. The handler should see the current state and skip the event.
- setNodeHealthState(datanode, HddsProtos.NodeState.HEALTHY_READONLY);
++ setNodeHealthState(datanode, HddsProtos.NodeState.HEALTHY);
+ deadNodeHandler.onMessage(datanode, publisher);
+
+ verify(publisher, times(0)).fireEvent(SCMEvents.REPLICATION_MANAGER_NOTIFY, datanode);
+ verify(deletedBlockLog, times(0)).onDatanodeDead(datanode.getID());
}
private void setNodeHealthState(DatanodeDetails datanode,
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java
index 4578e4529df,1280ba045e6..bb4f67b8b46
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java
@@@ -17,10 -17,16 +17,15 @@@
package org.apache.hadoop.hdds.scm.node;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+ import static org.mockito.ArgumentMatchers.any;
+ import static org.mockito.Mockito.doThrow;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
@@@ -36,13 -43,17 +42,15 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+ import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+ import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
-import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint;
import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils;
-import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
import org.apache.hadoop.util.Time;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@@ -56,6 -69,14 +64,8 @@@ public class TestNodeStateManager
private NodeStateManager nsm;
private ConfigurationSource conf;
private MockEventPublisher eventPublisher;
+ private SCMContext scmContext;
+ private NetworkTopology clusterMap;
- private LayoutVersionManager mockVersionManager;
- private int scmSlv;
- private int scmMlv;
@BeforeEach
public void setUp() {
@@@ -75,11 -96,19 +85,10 @@@
return new char[0];
}
};
-- // Make NodeStateManager behave as if SCM has completed finalization,
-- // unless a test changes the value of this variable.
- SCMContext scmContext = SCMContext.emptyContext();
+ scmContext = SCMContext.emptyContext();
- scmContext.setFinalizationCheckpoint(
- FinalizationCheckpoint.FINALIZATION_COMPLETE);
eventPublisher = new MockEventPublisher();
- nsm = new NodeStateManager(conf, eventPublisher, scmContext);
- scmSlv = maxLayoutVersion();
- scmMlv = maxLayoutVersion();
- mockVersionManager = mock(HDDSLayoutVersionManager.class);
- when(mockVersionManager.getMetadataLayoutVersion()).thenReturn(scmMlv);
- when(mockVersionManager.getSoftwareLayoutVersion()).thenReturn(scmSlv);
+ clusterMap = new NetworkTopologyImpl(new OzoneConfiguration());
- nsm = new NodeStateManager(conf, eventPublisher, clusterMap, mockVersionManager, scmContext);
++ nsm = new NodeStateManager(conf, eventPublisher, clusterMap, scmContext);
}
@Test
@@@ -201,11 -230,109 +210,82 @@@
assertEquals(SCMEvents.STALE_NODE, eventPublisher.getLastEvent());
dni.updateLastHeartbeatTime();
nsm.checkNodesHealth();
- assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth());
- assertEquals(SCMEvents.HEALTHY_READONLY_NODE, eventPublisher.getLastEvent());
-
- // Another health check run should move the node to healthy since its
- // metadata layout version matches SCM's.
- nsm.checkNodesHealth();
assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth());
- assertEquals(SCMEvents.HEALTHY_READONLY_TO_HEALTHY_NODE, eventPublisher.getLastEvent());
+ assertEquals(SCMEvents.UNHEALTHY_TO_HEALTHY_NODE, eventPublisher.getLastEvent());
eventPublisher.clearEvents();
-
- // Test how node state manager handles datanodes with lower metadata
- // layout version based on SCM's finalization checkpoint.
- dni.updateLastKnownLayoutVersion(
- UpgradeUtils.toLayoutVersionProto(scmMlv - 1, scmSlv));
- for (FinalizationCheckpoint checkpoint: FinalizationCheckpoint.values()) {
- scmContext.setFinalizationCheckpoint(checkpoint);
- LOG.info("Testing datanode state from current SCM finalization " +
- "checkpoint: {}", checkpoint);
- nsm.checkNodesHealth();
-
- // Datanodes should not be moved to healthy readonly until the SCM has
- // finished updating its metadata layout version as part of finalization.
- if (checkpoint.hasCrossed(FinalizationCheckpoint.MLV_EQUALS_SLV)) {
- assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth());
- assertEquals(SCMEvents.HEALTHY_READONLY_NODE, eventPublisher.getLastEvent());
- } else {
- assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth());
- assertNull(eventPublisher.getLastEvent());
- }
- }
}
+ @Test
+ public void testDeadNodeIsRemovedFromTopologyUntilItRecovers()
+ throws NodeAlreadyExistsException, NodeNotFoundException {
+ long now = Time.monotonicNow();
+ long staleLimit = HddsServerUtil.getStaleNodeInterval(conf) + 1000;
+ long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000;
+
+ // Like SCMNodeManager, add the node to the topology before adding it to NodeStateManager.
+ DatanodeDetails dn = generateDatanode();
+ dn.setNetworkName(dn.getUuidString());
+ clusterMap.add(dn);
- nsm.addNode(dn, defaultLayoutVersionProto());
++ nsm.addNode(dn, defaultVersionProto());
+ DatanodeInfo dni = nsm.getNode(dn);
+ String path = dn.getNetworkFullPath();
+
+ dni.updateLastHeartbeatTime(now - staleLimit);
+ nsm.checkNodesHealth();
+ assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth());
+ assertNotNull(clusterMap.getNode(path));
+
+ dni.updateLastHeartbeatTime(now - deadLimit);
+ nsm.checkNodesHealth();
+ assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth());
+ assertNull(clusterMap.getNode(path));
+
+ dni.updateLastHeartbeatTime();
+ nsm.checkNodesHealth();
- assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth());
++ assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth());
+ assertNotNull(clusterMap.getNode(path));
+ }
+
+ @Test
+ public void testReregisteredNodeIsRemovedFromTopologyWhenDead()
+ throws NodeAlreadyExistsException, NodeNotFoundException {
+ DatanodeDetails dn = generateDatanode();
+ dn.setNetworkName(dn.getUuidString());
+ clusterMap.add(dn);
- nsm.addNode(dn, defaultLayoutVersionProto());
++ nsm.addNode(dn, defaultVersionProto());
+ String path = dn.getNetworkFullPath();
+
+ // Re-registering with a new version or new ports replaces the DatanodeInfo with a copy that has no parent.
+ DatanodeDetails reregistered = new DatanodeDetails(dn);
+ reregistered.setParent(null);
- nsm.updateNode(reregistered, defaultLayoutVersionProto());
++ nsm.updateNode(reregistered, defaultVersionProto());
+
+ nsm.getNode(dn).updateLastHeartbeatTime(Time.monotonicNow() - HddsServerUtil.getDeadNodeInterval(conf) - 1000);
+ nsm.checkNodesHealth();
+ nsm.checkNodesHealth();
+
+ assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth());
+ assertNull(clusterMap.getNode(path));
+ }
+
+ @Test
+ public void testTopologyUpdateFailureDoesNotFailHealthCheck()
+ throws NodeAlreadyExistsException, NodeNotFoundException {
+ NetworkTopology failingClusterMap = mock(NetworkTopology.class);
+ when(failingClusterMap.contains(any())).thenReturn(true);
+ doThrow(new IllegalStateException("injected")).when(failingClusterMap).remove(any());
- nsm = new NodeStateManager(conf, eventPublisher, failingClusterMap, mockVersionManager, scmContext);
++ nsm = new NodeStateManager(conf, eventPublisher, failingClusterMap, scmContext);
+
+ DatanodeDetails dn = generateDatanode();
- nsm.addNode(dn, defaultLayoutVersionProto());
++ nsm.addNode(dn, defaultVersionProto());
+ nsm.getNode(dn).updateLastHeartbeatTime(Time.monotonicNow() - HddsServerUtil.getDeadNodeInterval(conf) - 1000);
+ nsm.checkNodesHealth();
+ nsm.checkNodesHealth();
+
+ assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth());
+ assertEquals(SCMEvents.DEAD_NODE, eventPublisher.getLastEvent());
+ }
+
@Test
public void testNodeOpStateCanBeSet()
throws NodeAlreadyExistsException, NodeNotFoundException {
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index aadc818769b,522fb17e1fb..e829de028e7
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -2375,9 -2393,11 +2386,11 @@@ public class RpcClient implements Clien
@Override
public OzoneMultipartUploadList listMultipartUploads(String volumeName,
String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
OmMultipartUploadList omMultipartUploadList;
- if (omVersion.compareTo(OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION) >= 0) {
+ if (OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION.isSupportedBy(omVersion)) {
omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker,
uploadIdMarker, maxUploads, true);
} else {
diff --cc hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java
index ca985fc993b,c7c4d343f6c..21a31808f06
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java
@@@ -325,18 -252,20 +328,32 @@@ public class TestRpcClient
}
}
+ @ParameterizedTest
+ @CsvSource({"invalid/volume, bucket1, INVALID_VOLUME_NAME", "volume1, invalid/bucket, INVALID_BUCKET_NAME"})
+ public void testListMultipartUploadsValidatesNames(String volume, String bucket, ResultCodes expected)
+ throws Exception {
+ RpcClient client = createRpcClient();
+ try {
+ OMException error = assertThrows(OMException.class,
+ () -> client.listMultipartUploads(volume, bucket, null, null, null, 10));
+ assertThat(error.getResult()).isEqualTo(expected);
+ } finally {
+ client.close();
+ }
+ }
+
+ private static ServiceInfo om(OzoneManagerVersion version) {
+ return new ServiceInfo.Builder()
+ .setNodeType(HddsProtos.NodeType.OM)
+ .setHostname("localhost")
+ .setOmVersion(version)
+ .build();
+ }
+
+ private static ServiceInfoEx serviceInfoEx(ServiceInfo... serviceInfos) {
+ return new ServiceInfoEx(Arrays.asList(serviceInfos), null, null);
+ }
+
private static RpcClient createRpcClient() throws IOException {
OzoneConfiguration config = new OzoneConfiguration();
return new RpcClient(config, null) {
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 5380f3ce4f0,e118ef68b41..0e457553e79
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@@ -264,13 -264,10 +264,15 @@@ public final class OmUtils
case GetSnapshotInfo:
case GetObjectTagging:
case GetBucketTagging:
+ case GetBucketDeletedBytes:
+ return true;
case GetQuotaRepairStatus:
case StartQuotaRepair:
+ // Prepare and CancelPrepare are now no-ops, but still produce responses for compatability.
+ // They are marked as read-only since they don't modify state anymore.
+ case Prepare:
+ case CancelPrepare:
+ case QueryUpgradeStatus:
case GetLifecycleConfiguration:
case GetLifecycleServiceStatus:
return true;
diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java
index 720ba53fee0,279889d930d..06c95559b28
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java
@@@ -569,14 -637,16 +637,16 @@@ public class TestHadoopRpcOMFollowerRea
GetKeyInfoRequest.Builder req = GetKeyInfoRequest.newBuilder()
.setKeyArgs(keyArgs);
- OMRequest omRequest = OMRequest.newBuilder()
+ OMRequest.Builder omRequest = OMRequest.newBuilder()
- .setVersion(ClientVersion.CURRENT_VERSION)
+ .setVersion(ClientVersion.CURRENT.serialize())
.setClientId(ClientId.randomId().toString())
.setCmdType(Type.GetKeyInfo)
- .setGetKeyInfoRequest(req)
- .build();
+ .setGetKeyInfoRequest(req);
+ if (readConsistency != null) {
+ omRequest.setReadConsistencyHint(readConsistency.getHint());
+ }
- client.submitRequest(null, omRequest);
+ client.submitRequest(null, omRequest.build());
}
private void assertHandledBy(int omNodeIdx) {
diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
index 0a0e45dd37d,79d524f004c..1ee4279c287
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
@@@ -373,6 -374,124 +374,124 @@@ public class TestS3GrpcOmTransport
}
@Test
+ public void testLeaderOnlyReadOverridesDefaultFollowerRead() throws Exception {
+ conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true);
+ configureHaOmService("om0", "om1");
+
+ AtomicInteger leaderRequestCount = new AtomicInteger();
+ AtomicInteger followerRequestCount = new AtomicInteger();
+ AtomicReference<OMRequest> leaderRequest = new AtomicReference<>();
+
+ client = new GrpcOmTransport(conf, ugi, omServiceId);
+ client.startClient("om0", createNodeChannel("om0",
+ leaderRequestCount, leaderRequest));
+ client.startClient("om1", createNodeChannel("om1",
+ followerRequestCount, new AtomicReference<>()));
+ client.changeLeaderProxyForTest("om0");
+ client.changeFollowerReadInitialProxy("om1");
+
+ client.submitRequest(OMRequest.newBuilder()
+ .setCmdType(Type.ListVolume)
- .setVersion(CURRENT_VERSION)
++ .setVersion(CURRENT.serialize())
+ .setClientId("test")
+ .setS3Authentication(S3Authentication.newBuilder()
+ .setAccessId("access-id")
+ .setSignature("signature")
+ .setStringToSign("string-to-sign"))
+ .setReadConsistencyHint(ReadConsistencyHint.newBuilder()
+ .setReadConsistency(ReadConsistencyProto.LINEARIZABLE_LEADER_ONLY))
+ .build());
+
+ assertEquals(1, leaderRequestCount.get());
+ assertEquals(0, followerRequestCount.get());
+ assertEquals(ReadConsistencyProto.LINEARIZABLE_LEADER_ONLY,
+ leaderRequest.get().getReadConsistencyHint().getReadConsistency());
+
+ client.submitRequest(OMRequest.newBuilder()
+ .setCmdType(Type.ListVolume)
- .setVersion(CURRENT_VERSION)
++ .setVersion(CURRENT.serialize())
+ .setClientId("test")
+ .setReadConsistencyHint(ReadConsistencyHint.newBuilder()
+ .setReadConsistency(ReadConsistencyProto.LINEARIZABLE_ALLOW_FOLLOWER))
+ .build());
+
+ assertEquals(1, followerRequestCount.get());
+ }
+
+ @Test
+ public void testLocalLeaseReadAvoidsKnownLeader() throws Exception {
+ conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true);
+ configureHaOmService("om0", "om1");
+
+ AtomicInteger leaderRequestCount = new AtomicInteger();
+ AtomicInteger followerRequestCount = new AtomicInteger();
+
+ client = new GrpcOmTransport(conf, ugi, omServiceId);
+ client.startClient("om0", createNodeChannel("om0",
+ leaderRequestCount, new AtomicReference<>()));
+ client.startClient("om1", createNodeChannel("om1",
+ followerRequestCount, new AtomicReference<>()));
+ client.changeLeaderProxyForTest("om0");
+ client.changeFollowerReadInitialProxy("om0");
+
+ client.submitRequest(OMRequest.newBuilder()
+ .setCmdType(Type.ListVolume)
- .setVersion(CURRENT_VERSION)
++ .setVersion(CURRENT.serialize())
+ .setClientId("test")
+ .setS3Authentication(S3Authentication.newBuilder()
+ .setAccessId("access-id")
+ .setSignature("signature")
+ .setStringToSign("string-to-sign")
+ .setSessionToken("session-token"))
+ .setReadConsistencyHint(ReadConsistencyHint.newBuilder()
+ .setReadConsistency(ReadConsistencyProto.LOCAL_LEASE))
+ .build());
+
+ assertEquals(0, leaderRequestCount.get());
+ assertEquals(1, followerRequestCount.get());
+ }
+
+ @Test
+ public void testExplicitFollowerReadWhenDisabledByDefault() throws Exception {
+ conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, false);
+ configureHaOmService("om0", "om1");
+
+ AtomicInteger leaderRequestCount = new AtomicInteger();
+ AtomicInteger followerRequestCount = new AtomicInteger();
+ AtomicReference<OMRequest> leaderRequest = new AtomicReference<>();
+ AtomicReference<OMRequest> followerRequest = new AtomicReference<>();
+
+ client = new GrpcOmTransport(conf, ugi, omServiceId);
+ client.startClient("om0", createNodeChannel("om0",
+ leaderRequestCount, leaderRequest));
+ client.startClient("om1", createNodeChannel("om1",
+ followerRequestCount, followerRequest));
+ client.changeLeaderProxyForTest("om0");
+ client.changeFollowerReadInitialProxy("om1");
+
+ OMRequest request = OMRequest.newBuilder()
+ .setCmdType(Type.ListVolume)
- .setVersion(CURRENT_VERSION)
++ .setVersion(CURRENT.serialize())
+ .setClientId("test")
+ .build();
+ client.submitRequest(request);
+
+ assertEquals(1, leaderRequestCount.get());
+ assertEquals(0, followerRequestCount.get());
+ assertEquals(ReadConsistencyProto.DEFAULT,
+ leaderRequest.get().getReadConsistencyHint().getReadConsistency());
+
+ client.submitRequest(request.toBuilder()
+ .setReadConsistencyHint(ReadConsistencyHint.newBuilder()
+ .setReadConsistency(ReadConsistencyProto.LOCAL_LEASE))
+ .build());
+
+ assertEquals(1, followerRequestCount.get());
+ assertEquals(ReadConsistencyProto.LOCAL_LEASE,
+ followerRequest.get().getReadConsistencyHint().getReadConsistency());
+ }
+
+ @Test
public void testFollowerReadFallsBackToLeaderOnNotLeaderException() throws Exception {
conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true);
configureHaOmService("om0", "om1");
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 278fda0f5ee,6a431e477b9..ca56caa2191
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@@ -352,12 -358,11 +360,12 @@@ public class TestBlockDataStreamOutput
}
}
+ @Unhealthy("Requires HDDS-16044 to finish implementing datanode version passing to client.")
@ParameterizedTest
@MethodSource("clientParameters")
- public void testDatanodeVersion(boolean flushDelay, boolean putBlockOnCloseEnabled)
+ public void testDatanodeVersion(boolean flushDelay, boolean putBlockWithoutRaft)
throws Exception {
- OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled);
+ OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockWithoutRaft);
try (OzoneClient client = newClient(cluster.getConf(), config)) {
// Verify all DNs internally have versions set correctly
List<HddsDatanodeService> dns = cluster.getHddsDatanodes();
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 8561b8e4aa2,a781d824a57..412e2814dfd
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@@ -102,9 -103,7 +103,8 @@@ import org.apache.hadoop.ozone.om.servi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ToolRunner;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.tag.Unhealthy;
import org.apache.ratis.server.RaftServerConfigKeys;
- import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@@ -2580,60 -2562,19 +2565,67 @@@ public class TestOzoneShellHA
execute(ozoneShell, args);
}
+ /**
+ * Test that even though prepare commands are deprecated, they still succeed without blocking writes.
+ */
+ @Test
+ public void testPrepareCommandNoOp() throws IOException {
+ String volumeName = "vol-prepare-test";
+ String bucketName = "bucket-prepare-test";
+
+ // Execute prepare command via CLI
+ String[] args = new String[] {"om", "prepare", "--service-id", omServiceId};
+ execute(ozoneAdminShell, args);
+ out.reset();
+
+ // Verify write operations still work (prepare should not block them)
+ // Create volume
+ args = new String[] {"volume", "create", "o3://" + omServiceId + OZONE_URI_DELIMITER + volumeName};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Create bucket
+ args = new String[] {"bucket", "create", "o3://" + omServiceId + OZONE_URI_DELIMITER + volumeName +
+ OZONE_URI_DELIMITER + bucketName};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Create key
+ String keyName = OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER +
+ bucketName + OZONE_URI_DELIMITER + "testkey";
+ args = new String[] {"key", "put", "o3://" + omServiceId + keyName, testFile.getPath()};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Verify the key was created successfully
+ OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ assertNotNull(bucket.getKey("testkey"), "Key should be created successfully after prepare");
+
+ // Execute cancelprepare command via CLI
+ args = new String[] {"om", "cancelprepare", "--service-id", omServiceId};
+ execute(ozoneAdminShell, args);
+ out.reset();
+
+ // Cleanup
+ args = new String[] {"volume", "delete", volumeName, "-r", "--yes"};
+ execute(ozoneShell, args);
+ out.reset();
+ }
+
- private static String getKeyProviderURI(MiniKMS kms) {
- if (kms == null) {
- return "";
+ private static String getKeyProviderURI() {
+ File jks = new File(kmsDir, "kms.jks");
+ return JavaKeyStoreProvider.SCHEME_NAME + "://file" + jks.toURI().getPath();
+ }
+
+ private static void seedEncryptionKey(OzoneConfiguration conf) throws Exception {
+ KeyProvider provider = KeyProviderFactory.get(new URI(getKeyProviderURI()), conf);
+ if (provider.getMetadata(ENCRYPTION_KEY_NAME) == null) {
+ KeyProvider.Options options = KeyProvider.options(conf);
+ options.setBitLength(128);
+ provider.createKey(ENCRYPTION_KEY_NAME, options);
+ provider.flush();
}
- return KMSClientProvider.SCHEME_NAME + "://" +
- kms.getKMSUrl().toExternalForm().replace("://", "@");
}
protected MiniOzoneHAClusterImpl getCluster() {
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index e2ed4e40231,43f7c6aae51..1445e1bb2bf
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -171,9 -171,7 +171,10 @@@ enum Type
DeleteRevokedSTSTokens = 155;
GetCallerIdentity = 156;
SnapshotCount = 157;
- StartFinalizeUpgrade = 158;
- QueryUpgradeStatus = 159;
- CompleteFinalizeUpgrade = 160;
+ GetBucketDeletedBytes = 158;
++ StartFinalizeUpgrade = 159;
++ QueryUpgradeStatus = 160;
++ CompleteFinalizeUpgrade = 161;
}
enum SafeMode {
@@@ -340,9 -338,7 +341,10 @@@ message OMRequest
optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 157;
optional GetCallerIdentityRequest getCallerIdentityRequest = 158;
optional SnapshotCountRequest snapshotCountRequest = 159;
- optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 160;
- optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 161;
- optional CompleteFinalizeUpgradeRequest completeFinalizeUpgradeRequest = 162;
+ optional GetBucketDeletedBytesRequest getBucketDeletedBytesRequest = 160;
++ optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 161;
++ optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 162;
++ optional CompleteFinalizeUpgradeRequest completeFinalizeUpgradeRequest = 163;
}
message OMResponse {
@@@ -492,9 -489,7 +494,10 @@@
optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 155;
optional GetCallerIdentityResponse getCallerIdentityResponse = 156;
optional SnapshotCountResponse snapshotCountResponse = 157;
- optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 158;
- optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 159;
- optional CompleteFinalizeUpgradeResponse completeFinalizeUpgradeResponse = 160;
+ optional GetBucketDeletedBytesResponse getBucketDeletedBytesResponse = 158;
++ optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 159;
++ optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 160;
++ optional CompleteFinalizeUpgradeResponse completeFinalizeUpgradeResponse = 161;
}
enum Status {
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 1d0d18235ea,9b22579ec09..45d4129dcba
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -312,9 -319,10 +319,10 @@@ import org.apache.hadoop.ozone.om.servi
import org.apache.hadoop.ozone.om.service.QuotaRepairTask;
import org.apache.hadoop.ozone.om.service.RevokedSTSTokenCleanupService;
import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService;
+ import org.apache.hadoop.ozone.om.snapshot.trapped.BucketDeletedDataCalculator;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
-import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizer;
+import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizeService;
+import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
index 50b1d25b30d,5e4bb9ca360..4abdc1cc683
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
@@@ -50,13 -51,12 +50,10 @@@ import org.apache.hadoop.ozone.om.lock.
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.response.DummyOMClientResponse;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
- import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler;
import org.apache.hadoop.ozone.protocolPB.RequestHandler;
- import org.apache.hadoop.ozone.security.STSSecurityUtil;
- import org.apache.hadoop.ozone.security.STSTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.ratis.proto.RaftProtos;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java
index c5cdda6ec97,ea68fa902be..1fe0a266008
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java
@@@ -17,33 -17,29 +17,30 @@@
package org.apache.hadoop.ozone.om.response.upgrade;
-import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.APPARENT_VERSION_KEY;
- import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE;
import java.io.IOException;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
- import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Response for finalizeUpgrade request.
+ * Response for completing OM finalization (submitted by OMUpgradeFinalizeService).
*/
- @CleanupTableInfo(cleanupTables = {META_TABLE})
-public class OMFinalizeUpgradeResponse extends OMClientResponse {
+public class OMCompleteFinalizeUpgradeResponse extends OMClientResponse {
private static final Logger LOG =
- LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class);
- private int layoutVersionToWrite = -1;
+ LoggerFactory.getLogger(OMCompleteFinalizeUpgradeResponse.class);
+ private int serializedApparentVersion = -1;
- public OMFinalizeUpgradeResponse(
+ public OMCompleteFinalizeUpgradeResponse(
OzoneManagerProtocolProtos.OMResponse omResponse,
- int layoutVersionToWrite) {
+ int serializedApparentVersion) {
super(omResponse);
- this.layoutVersionToWrite = layoutVersionToWrite;
+ this.serializedApparentVersion = serializedApparentVersion;
}
@Override
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java
index 6687e5a6950,ea68fa902be..4f257546cbd
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java
@@@ -17,13 -17,11 +17,10 @@@
package org.apache.hadoop.ozone.om.response.upgrade;
- import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE;
-import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY;
--
import java.io.IOException;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
- import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.slf4j.Logger;
@@@ -32,24 -30,16 +29,23 @@@ import org.slf4j.LoggerFactory
/**
* Response for finalizeUpgrade request.
*/
- @CleanupTableInfo(cleanupTables = {META_TABLE})
-public class OMFinalizeUpgradeResponse extends OMClientResponse {
- private static final Logger LOG =
- LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class);
- private int layoutVersionToWrite = -1;
+public class OMStartFinalizeUpgradeResponse extends OMClientResponse {
+ private static final Logger LOG = LoggerFactory.getLogger(OMStartFinalizeUpgradeResponse.class);
+
+ private final boolean finalizationNeeded;
+
+ public OMStartFinalizeUpgradeResponse(OzoneManagerProtocolProtos.OMResponse omResponse) {
+ this(omResponse, true);
+ }
- public OMFinalizeUpgradeResponse(
- OzoneManagerProtocolProtos.OMResponse omResponse,
- int layoutVersionToWrite) {
+ /**
+ * @param finalizationNeeded whether OM still needs to finalize. When {@code false} the finalization-in-progress
+ * marker is not persisted, so that initiating finalize on an already-finalized cluster does not orphan the key
+ * in the DB (the async {@code OMUpgradeFinalizeService} would never clear it since finalization is not needed).
+ */
+ public OMStartFinalizeUpgradeResponse(OzoneManagerProtocolProtos.OMResponse omResponse, boolean finalizationNeeded) {
super(omResponse);
- this.layoutVersionToWrite = layoutVersionToWrite;
+ this.finalizationNeeded = finalizationNeeded;
}
@Override
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index aef44dd73f8,2e234ac76ec..e1795f9dd95
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -60,7 -60,7 +60,8 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper;
import org.apache.hadoop.hdds.utils.FaultInjector;
+ import org.apache.hadoop.hdds.utils.db.cache.TableCacheUpdateTracker;
+import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OzoneAclUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
@@@ -266,9 -267,15 +270,15 @@@ public class OzoneManagerRequestHandle
break;
case ListOpenFiles:
ListOpenFilesResponse listOpenFilesResponse = listOpenFiles(
- request.getListOpenFilesRequest(), request.getVersion());
+ request.getListOpenFilesRequest(), clientVersion);
responseBuilder.setListOpenFilesResponse(listOpenFilesResponse);
break;
+ case GetBucketDeletedBytes:
+ GetBucketDeletedBytesResponse getBucketDeletedBytesResponse =
+ getBucketDeletedBytes(request.getGetBucketDeletedBytesRequest());
+ responseBuilder.setGetBucketDeletedBytesResponse(
+ getBucketDeletedBytesResponse);
+ break;
case ServiceList:
ServiceListResponse serviceListResponse = getServiceList(
request.getServiceListRequest());
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
index e2b66ffa228,c5640b92c01..7a0657828af
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
@@@ -74,6 -87,10 +86,7 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareRequestArgs;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org