You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by er...@apache.org on 2026/07/16 20:57:37 UTC
(ozone) 01/01: Merge branch 'master' into HDDS-14496-zdu-master-3
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e16d646d29bda26bd5da9d98ba9c2e828348dfac
Merge: 98c55fe4905 6ac2289aafd
Author: Ethan Rose <er...@apache.org>
AuthorDate: Thu Jul 16 16:51:04 2026 -0400
Merge branch 'master' into HDDS-14496-zdu-master-3
* master: (53 commits)
HDDS-15876. Remove ozone-csi (#10780)
HDDS-15508. Add 2.2.0 to compatibility/upgrade test (#10782)
HDDS-15867. [JDK25] Bump byte-buddy to 1.18.11 (#10776)
HDDS-15809. Validate x-amz-copy-source-range on UploadPartCopy (#10711).
HDDS-15678. OFS isDirectory/isFile should not trigger pipeline refresh or return block locations (#10741)
HDDS-15171. Add available space check on follower during bootstrap. (#10185)
HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs (#10209)
HDDS-15709. Replace random name suffix with uniqueObjectName (#10773)
HDDS-15849. Force channel shutdown in XceiverClientGrpc.close (#10747)
HDDS-15765. Fail fast when --node-id is omitted for OM compact/defrag on an HA cluster (#10699)
HDDS-14989. Delay follower SCM DN server start until Ratis log catch-up. (#10617).
HDDS-15456. Add SCM DB lookup and identify orphan(wrt SCM) and deleted-but-present containers (#10547)
HDDS-15812. Rename test utils for naming convention (#10766)
HDDS-15600. Fix ListObjects response for encoding-type and empty delimiter (#10586).
HDDS-15656. Fix non-atomic data size accumulation in ContainerBalancer move callback (#10708)
HDDS-15667. `containerbalancer status`: show stop reason and iteration details after stop (#10681)
HDDS-15823. Fix flaky TestFailureHandlingByClient#testContainerExclusionWithClosedContainerException (#10721)
HDDS-15837. Add health-aware container ID listing in SCM (#10736)
HDDS-15851. Return 403 for expired or out-of-range pre-signed S3 URLs (#10749)
HDDS-15836. Add Archiver.appendFile for incremental TAR writes (#10735)
...
Conflicts:
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
hadoop-ozone/dist/src/main/compose/upgrade/test.sh
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
.github/dependabot.yml | 56 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
.run/CsiServer.run.xml | 33 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 40 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 9 +-
.../TestXceiverClientManagerCloseContention.java | 161 +++
.../scm/storage/TestStreamBlockInputStream.java | 156 +++
hadoop-hdds/common/src/main/conf/ozone-env.sh | 11 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 1 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +
.../common/src/main/resources/ozone-default.xml | 23 +
.../commandhandler/DeleteBlocksCommandHandler.java | 7 +
.../TestDeleteBlocksCommandHandler.java | 108 ++
...achine.java => ContainerStateMachineTests.java} | 4 +-
.../ratis/TestContainerStateMachineFollower.java | 2 +-
.../ratis/TestContainerStateMachineLeader.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 16 +-
...ersAbstract.java => ContainerScannerTests.java} | 2 +-
.../TestBackgroundContainerDataScanner.java | 2 +-
.../TestBackgroundContainerMetadataScanner.java | 2 +-
.../ozoneimpl/TestOnDemandContainerScanner.java | 2 +-
.../docs/content/design/efficient-snapdiff.md | 254 ++++
hadoop-hdds/docs/content/interface/CSI.md | 93 --
hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes
hadoop-hdds/docs/content/interface/CSI.zh.md | 92 --
hadoop-hdds/docs/content/tools/Admin.md | 4 +-
hadoop-hdds/docs/content/tools/Repair.md | 4 +-
.../{TestCoderBase.java => CoderTests.java} | 2 +-
...estRSRawCoderBase.java => RSRawCoderTests.java} | 4 +-
.../{TestRawCoderBase.java => RawCoderTests.java} | 6 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 2 +-
.../rawcoder/TestNativeXORRawCoder.java | 2 +-
.../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 2 +-
...tXORRawCoderBase.java => XORRawCoderTests.java} | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 76 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +-
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../interface-admin/src/main/resources/proto.lock | 18 +
.../utils/{TestUtils.java => RocksTestUtils.java} | 4 +-
.../utils/db/TestManagedRawSSTFileIterator.java | 4 +-
.../hadoop/hdds/utils/db/TestSstFileSetReader.java | 6 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 9 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 +-
.../SCMDeletedBlockTransactionStatusManager.java | 133 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 4 +-
.../hdds/scm/container/ContainerManagerImpl.java | 5 +-
.../hdds/scm/container/ContainerStateManager.java | 5 +-
.../scm/container/ContainerStateManagerImpl.java | 5 +-
.../scm/container/balancer/ContainerBalancer.java | 28 +-
.../balancer/ContainerBalancerStatusInfo.java | 43 +-
.../balancer/ContainerBalancerStopReason.java | 63 +
.../container/balancer/ContainerBalancerTask.java | 143 +-
.../scm/container/states/ContainerStateMap.java | 49 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 5 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 135 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +-
.../hdds/scm/ha/StatefulServiceDefinition.java} | 38 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 83 +-
.../hdds/scm/security/RootCARotationManager.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +-
.../scm/container/TestContainerStateManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 43 +
.../balancer/TestContainerBalancerStatusInfo.java | 66 +
.../balancer/TestContainerBalancerTask.java | 58 +
...andler.java => MisReplicationHandlerTests.java} | 2 +-
.../replication/TestECMisReplicationHandler.java | 2 +-
.../TestRatisMisReplicationHandler.java | 2 +-
.../container/states/TestContainerStateMap.java | 102 +-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +-
.../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 5 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +-
.../ozone/admin/om/snapshot/DefragSubCommand.java | 11 +-
.../datanode/TestContainerBalancerSubCommand.java | 242 +++-
.../admin/om/snapshot/TestDefragSubCommand.java | 22 +
.../container/analyze/AnalyzeSubcommand.java | 234 +++-
.../analyze/ScmContainerMetadataReader.java | 155 ++
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 51 +-
.../ozone/debug/replicas/ReplicasVerify.java | 112 +-
.../analyze/ContainerAnalyzeTestHelper.java | 37 +
.../container/analyze/TestAnalyzeSubcommand.java | 188 ++-
.../analyze/TestScmContainerMetadataReader.java | 100 ++
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 11 +-
.../hadoop/ozone/repair/om/TestCompactOMDB.java | 12 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../ozone/client/protocol/ClientProtocol.java | 22 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
.../hadoop/ozone/client/TestOzoneBucket.java | 72 +
.../client/TestRpcClientGetFileStatusHeadOp.java | 93 ++
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 22 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 +
.../hadoop/ozone/om/helpers/AclListBuilder.java | 7 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 74 +-
.../ozone/om/helpers/OmMultipartAbortInfo.java | 42 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 17 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 22 +
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 55 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../ozone/security/acl/IAccessAuthorizer.java | 16 +-
.../csi/dev-support/findbugsExcludeFile.xml | 22 -
hadoop-ozone/csi/pom.xml | 290 ----
.../apache/hadoop/ozone/csi/ControllerService.java | 116 --
.../org/apache/hadoop/ozone/csi/CsiServer.java | 177 ---
.../apache/hadoop/ozone/csi/IdentityService.java | 72 -
.../org/apache/hadoop/ozone/csi/NodeService.java | 148 --
.../org/apache/hadoop/ozone/csi/package-info.java | 21 -
hadoop-ozone/csi/src/main/proto/csi.proto | 1323 -----------------
hadoop-ozone/csi/src/main/resources/proto.lock | 1479 --------------------
.../dev-support/intellij/ozone-site-ha.xml | 10 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
hadoop-ozone/dist/pom.xml | 5 -
.../dist/src/main/compose/common/replicas-test.sh | 2 +
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 20 -
.../src/main/compose/ozone-csi/docker-compose.yaml | 64 -
.../dist/src/main/compose/ozone-csi/docker-config | 44 -
.../dist/src/main/compose/ozone-csi/test.sh | 31 -
.../main/compose/ozonesecure-ha/s3-haproxy.yaml | 3 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 11 +-
.../dist/src/main/compose/xcompat/clients.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 53 -
.../main/k8s/definitions/ozone-csi/csi-crd.yaml | 21 -
.../main/k8s/definitions/ozone-csi/csi-node.yaml | 95 --
.../main/k8s/definitions/ozone-csi/csi-rbac.yaml | 66 -
.../definitions/ozone-csi/csi-storageclass.yaml | 20 -
.../k8s/definitions/ozone-csi/definitions/csi.yaml | 28 -
.../k8s/definitions/test-webserver/flekszible.yaml | 16 -
.../test-webserver/webserver-deployment.yaml | 50 -
.../test-webserver/webserver-service.yaml | 29 -
.../test-webserver/webserver-volume.yaml | 29 -
.../src/main/k8s/examples/ozone-dev/Flekszible | 7 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 6 +-
.../examples/ozone-dev/csi/csi-node-daemonset.yaml | 97 --
.../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 --
.../csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone-dev/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../examples/ozone-dev/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
.../dist/src/main/k8s/examples/ozone/Flekszible | 5 -
.../main/k8s/examples/ozone/config-configmap.yaml | 6 +-
.../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 97 --
.../examples/ozone/csi/csi-ozone-clusterrole.yaml | 98 --
.../ozone/csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../k8s/examples/ozone/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 2 -
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
hadoop-ozone/dist/src/main/smoketest/csi.robot | 34 -
.../src/main/smoketest/s3/MultipartUpload.robot | 15 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 7 -
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../AbstractTestStorageDistributionEndpoint.java | 14 +-
...ndpointUtil.java => ReconEndpointTestUtil.java} | 6 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 36 +-
hadoop-ozone/integration-test-s3/pom.xml | 7 +-
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 169 ++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 239 +++-
hadoop-ozone/integration-test/pom.xml | 5 -
...eldsBase.java => ConfigurationFieldsTests.java} | 8 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 44 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 89 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 +-
.../fs/ozone/TestOFSIsDirectoryBenchmark.java | 153 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 6 +-
.../hdds/scm/TestContainerReportWithKeys.java | 7 +-
.../TestSCMFollowerCatchupWithContainerReport.java | 336 +++++
.../TestReplicationManagerIntegration.java | 8 +-
.../node/{TestNodeUtil.java => NodeTestUtil.java} | 4 +-
.../scm/node/TestDecommissionAndMaintenance.java | 16 +-
.../hdds/scm/safemode/TestSafeModeSCMHA.java | 18 +-
...UpgradeUtils.java => HddsUpgradeTestUtils.java} | 6 +-
.../TestDNDataDistributionFinalization.java | 4 +-
.../TestScmDataDistributionFinalization.java | 14 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 18 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 9 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 25 +-
.../client/rpc/TestFailureHandlingByClient.java | 13 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 6 +-
...tInputStreamBase.java => InputStreamTests.java} | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 12 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 30 +-
.../rpc/read/TestStreamBlockInputStream.java | 12 +-
.../ozone/client/rpc/read/TestStreamRead.java | 6 +-
.../rpc/read/TestStreamReadDatanodeFailover.java | 4 +-
...estDuplicateContainerDirScannerIntegration.java | 1 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 4 +
....java => ContainerScannerIntegrationTests.java} | 2 +-
...tBackgroundContainerDataScannerIntegration.java | 2 +-
...kgroundContainerMetadataScannerIntegration.java | 2 +-
.../TestOnDemandContainerScannerIntegration.java | 2 +-
...estDataValidate.java => DataValidateTests.java} | 2 +-
.../freon/TestDataValidateWithDummyContainers.java | 2 +-
.../TestDataValidateWithSafeByteOperations.java | 2 +-
.../TestDataValidateWithUnsafeByteOperations.java | 2 +-
.../ozone/local/TestLocalOzoneClusterRuntime.java | 136 ++
.../ozone/om/AbstractOzoneManagerHATest.java | 11 +-
.../om/{TestBucket.java => BucketForTesting.java} | 16 +-
...d.java => OzoneManagerHAFollowerReadTests.java} | 2 +-
...zoneManagerHA.java => OzoneManagerHATests.java} | 2 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 8 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 12 +-
.../apache/hadoop/ozone/om/TestOMBootstrap.java | 5 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 80 +-
.../ozone/om/TestOMRatisSnapshotTransfer.java | 5 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 13 +-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 9 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 15 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 11 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 8 +-
.../ozone/om/service/TestBlockDeletionService.java | 4 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 6 +-
.../{TestOmSnapshot.java => OmSnapshotTests.java} | 47 +-
.../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 +
.../ozone/om/snapshot/TestOmSnapshotDisabled.java | 8 +-
.../om/snapshot/TestOmSnapshotDisabledRestart.java | 8 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 +-
.../TestOmSnapshotFsoWithoutNativeLib.java | 2 +-
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 2 +-
.../om/snapshot/TestOmSnapshotObjectStore.java | 2 +-
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 5 +-
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +-
...skBalancerDuringDecommissionAndMaintenance.java | 4 +-
.../ozone/shell/TestOzoneDebugReplicasVerify.java | 136 ++
.../src/main/proto/OmClientProtocol.proto | 4 +
.../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 +-
.../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 36 +
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 65 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 26 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 23 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 +++-
.../hadoop/ozone/om/request/OMClientRequest.java | 39 +-
.../om/request/key/OMAllocateBlockRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 3 -
.../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 42 +-
.../multipart/S3MultipartUploadAbortRequest.java | 42 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 9 +-
.../S3MultipartUploadCommitPartRequest.java | 74 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 8 +-
.../S3MultipartUploadCompleteRequest.java | 78 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 +-
.../om/request/util/OMMultipartUploadUtils.java | 102 ++
.../AbstractS3MultipartAbortResponse.java | 53 +-
.../S3ExpiredMultipartUploadsAbortResponse.java | 3 +-
.../multipart/S3MultipartUploadAbortResponse.java | 30 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 14 +-
.../S3MultipartUploadCommitPartResponse.java | 23 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 8 +-
.../S3MultipartUploadCompleteResponse.java | 15 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 +-
.../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +-
.../ozone/om/snapshot/SnapshotDiffValueParser.java | 359 +++++
.../protocolPB/OzoneManagerRequestHandler.java | 13 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 33 +
.../TestOmRatisSnapshotProvider.java | 59 +
.../TestOMClientRequestUserInfoFallback.java | 106 --
.../om/request/key/TestOMKeyCommitRequest.java | 2 -
.../TestS3MultipartUploadCommitPartRequest.java | 355 +++++
.../s3/multipart/S3MultipartResponseTests.java | 10 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 4 +-
.../om/snapshot/TestSnapshotDiffValueParser.java | 249 ++++
.../protocolPB/TestOzoneManagerRequestHandler.java | 83 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 86 ++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 67 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 11 +
.../TestBasicRootedOzoneClientAdapterHeadOp.java | 176 +++
.../fs/ozone/TestRootedOzoneFileSystemHeadOp.java | 174 +++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
hadoop-ozone/pom.xml | 1 -
.../webapps/recon/ozone-recon-web/.eslintignore | 26 -
.../recon/ozone-recon-web/eslint.config.mjs | 109 ++
.../webapps/recon/ozone-recon-web/package.json | 15 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1111 +++++++--------
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +-
.../ozone/s3/commontypes/RequestParameters.java | 11 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 7 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 5 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 13 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 37 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 7 +-
.../signature/AccessDeniedResourceException.java | 12 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 +
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 4 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 17 +
.../hadoop/ozone/s3/endpoint/TestRootList.java | 94 ++
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 12 +-
.../s3/signature/TestAWSSignatureProcessor.java | 37 +
.../signature/TestAuthorizationV4QueryParser.java | 16 +-
hadoop-ozone/tools/pom.xml | 12 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 294 +++-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 9 +-
pom.xml | 32 +-
342 files changed, 9512 insertions(+), 7428 deletions(-)
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 289b5db922a,a0f1c1bf362..8b0ea537d4e
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@@ -38,9 -38,9 +38,11 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND_COUNT_UPDATED;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NEW_NODE;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
+ import static org.apache.ozone.test.MetricsAsserts.getLongCounter;
+ import static org.apache.ozone.test.MetricsAsserts.getMetrics;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
index bfce2f29097,2db0df2db61..8235e18fd05
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
@@@ -17,8 -17,7 +17,7 @@@
package org.apache.hadoop.hdds.scm.node;
- import static java.lang.Thread.sleep;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
+import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager;
import static org.apache.ozone.test.MetricsAsserts.assertGauge;
import static org.apache.ozone.test.MetricsAsserts.getLongCounter;
import static org.apache.ozone.test.MetricsAsserts.getMetrics;
@@@ -38,9 -39,10 +37,10 @@@ import org.apache.hadoop.hdds.scm.HddsT
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+ import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@@ -238,8 -244,10 +238,9 @@@ public class TestSCMNodeMetrics
assertGauge("TotalFilesystemAvailable", 150L,
getMetrics(SCMNodeMetrics.class.getSimpleName()));
nodeManager.processHeartbeat(registeredDatanode);
- sleep(4000);
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5000);
metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
- assertGauge("InServiceHealthyReadonlyNodes", 0, metricsSource);
assertGauge("InServiceHealthyNodes", 1, metricsSource);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 77420a88216,ab4da4badd9..ca911f895c7
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@@ -134,7 -143,7 +143,7 @@@ public final class OmKeyInfo extends Wi
return new DelegatedCodec<>(
Proto2Codec.get(KeyInfo.getDefaultInstance()),
OmKeyInfo::getFromProtobuf,
- k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize()),
- k -> k.getProtobuf(true, ClientVersion.CURRENT_VERSION, isOpenKey),
++ k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize(), isOpenKey),
OmKeyInfo.class);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
index 2b882c55b7b,cf37681a793..39a3497aa1d
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
@@@ -55,7 -60,7 +60,7 @@@ public class RepeatedOmKeyInfo implemen
return new DelegatedCodec<>(
Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()),
RepeatedOmKeyInfo::getFromProto,
- k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize()),
- k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey),
++ k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize(), isOpenKey),
RepeatedOmKeyInfo.class);
}
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 3c079cd884f,e6c13160711..e9d0795db1e
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@@ -33,18 -33,16 +33,19 @@@ RESULT_DIR="$ALL_RESULT_DIR" create_res
# This is the version of Ozone that should use the runner image to run the
# code that was built. Other versions will pull images from docker hub.
- # run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION"
-run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
++# run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
+ # run_test ha non-rolling-upgrade 2.1.1 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION"
- #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
- #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
+ # run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
+ # run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.4.0 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.3.0 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.2.1 "$OZONE_CURRENT_VERSION"
# run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION"
- # Rolling upgrade test, commented out for now
- run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
++# Rolling upgrade test
++run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
+
generate_report "upgrade" "$ALL_RESULT_DIR"
exit "$RESULT"
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
index ae7e091658c,eea584643e6..3f735828b4c
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
@@@ -41,17 -56,26 +41,17 @@@ import org.slf4j.LoggerFactory
/**
* Helper methods for testing HDDS upgrade finalization in integration tests.
*/
- public final class TestHddsUpgradeUtils {
+ public final class HddsUpgradeTestUtils {
- private static final Logger LOG = LoggerFactory.getLogger(TestHddsUpgradeUtils.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HddsUpgradeTestUtils.class);
- private TestHddsUpgradeUtils() { }
- private static final ReplicationConfig RATIS_THREE =
- ReplicationConfig.fromProtoTypeAndFactor(HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.THREE);
-
+ private HddsUpgradeTestUtils() { }
- public static void waitForFinalizationFromClient(
- StorageContainerLocationProtocol scmClient, String clientID)
- throws Exception {
+ public static void waitForFinalizationFromClient(StorageContainerLocationProtocol scmClient) throws Exception {
LambdaTestUtils.await(60_000, 1_000, () -> {
- UpgradeFinalization.Status status = scmClient
- .queryUpgradeFinalizationProgress(clientID, true, true)
- .status();
- LOG.info("Waiting for upgrade finalization to complete from client." +
- " Current status is {}.", status);
- return status == FINALIZATION_DONE || status == ALREADY_FINALIZED;
+ HddsProtos.UpgradeStatus status = scmClient.queryUpgradeStatus();
+ LOG.info("Waiting for upgrade finalization to complete from client. Current status is:\n{}", status);
+ return status.getShouldFinalize();
});
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
index 76bab76d66b,8c7daace078..a33af09e11f
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
@@@ -154,12 -167,24 +154,12 @@@ public class TestDNDataDistributionFina
// Validate pre-finalization state
validatePreDataDistributionFeatureState();
- // Now trigger finalization
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
-
// Wait for finalization to complete
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Verify finalization completed
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
// Create more data and deletions to test post-finalization behavior
String keyName3 = "testKey3";
@@@ -190,10 -215,21 +190,10 @@@
out.write(data);
}
bucket.deleteKey(keyName);
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- // Wait for finalization
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
-
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
// Verify the system can handle scenarios where pendingDeleteBlockCount
// might be missing and needs recalculation
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
index 6707cd87f00,3fb1bc121af..81fb21ebcde
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
@@@ -26,9 -26,9 +26,9 @@@ import static org.apache.hadoop.hdds.Hd
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT;
import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.EMPTY_SUMMARY;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@@ -160,18 -191,20 +160,18 @@@ public class TestScmDataDistributionFin
@Test
@Flaky("HDDS-14050")
public void testFinalizationEmptyClusterDataDistribution() throws Exception {
- init(new OzoneConfiguration(), null, true);
+ init(new OzoneConfiguration());
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
@@@ -266,15 -299,26 +266,15 @@@
flushDBTransactionBuffer(activeSCM);
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
index eff92e27589,c0e64ac4cac..802562f886a
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
@@@ -17,13 -17,21 +17,13 @@@
package org.apache.hadoop.hdds.upgrade;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmToFinalize;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmToFinalize;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
+import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@@ -90,54 -126,118 +90,54 @@@ public class TestScmHAFinalization
}
}
- /**
- * Argument supplier for parameterized tests.
- */
- public static Stream<Arguments> injectionPointsToTest() {
- // Do not test from BEFORE_PRE_FINALIZE_UPGRADE injection point.
- // Finalization will not have started so there will be no persisted state
- // to resume from.
- return Stream.of(
- Arguments.of(UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE),
- Arguments.of(UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION),
- Arguments.of(UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE)
- );
- }
-
- @ParameterizedTest
- @MethodSource(METHOD_SOURCE)
- public void testFinalizationWithLeaderChange(
- UpgradeTestInjectionPoints haltingPoint) throws Exception {
-
- CountDownLatch pauseLatch = new CountDownLatch(1);
- CountDownLatch unpauseLatch = new CountDownLatch(1);
- init(new OzoneConfiguration(),
- UpgradeTestUtils.newPausingFinalizationExecutor(haltingPoint,
- pauseLatch, unpauseLatch, LOG), 0);
- pauseLatch.await();
-
- // Stop the leader, forcing a leader change in the middle of finalization.
- // This will cause the initial client call for finalization
- // to be interrupted.
- StorageContainerManager oldLeaderScm = cluster.getActiveSCM();
- LOG.info("Stopping current SCM leader {} to initiate a leader change.",
- oldLeaderScm.getSCMNodeId());
- cluster.shutdownStorageContainerManager(oldLeaderScm);
-
- // Wait for the remaining two SCMs to elect a new leader.
- cluster.waitForClusterToBeReady();
-
- // While finalization is paused, check its state on the remaining SCMs.
- checkMidFinalizationConditions(haltingPoint,
- cluster.getStorageContainerManagersList());
-
- // Restart actually creates a new SCM.
- // Since this SCM will be a follower, the implementation of its upgrade
- // finalization executor does not matter for this test.
- cluster.restartStorageContainerManager(oldLeaderScm, true);
-
- // Make sure the original SCM leader is not the leader anymore.
- StorageContainerManager newLeaderScm = cluster.getActiveSCM();
- assertNotEquals(newLeaderScm.getSCMNodeId(),
- oldLeaderScm.getSCMNodeId());
-
- // Resume finalization from the new leader.
- unpauseLatch.countDown();
-
- // Client should complete exceptionally since the original SCM it
- // requested to was restarted.
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- // Make sure old leader has caught up and all SCMs have finalized.
- waitForScmsToFinalize(cluster.getStorageContainerManagersList());
-
- HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
- HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
- }
-
- @ParameterizedTest
- @MethodSource(METHOD_SOURCE)
- @Flaky("HDDS-8714")
- public void testFinalizationWithRestart(
- UpgradeTestInjectionPoints haltingPoint) throws Exception {
- CountDownLatch terminateLatch = new CountDownLatch(1);
- init(new OzoneConfiguration(),
- UpgradeTestUtils.newTerminatingFinalizationExecutor(haltingPoint,
- terminateLatch, LOG),
- 0);
- terminateLatch.await();
+ @Test
+ public void testFinalizedDatanodesShutDownWithPrefinalizedScm() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(SCMStorageConfig.TESTING_INIT_APPARENT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.serialize());
+ conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT, "5s");
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT, "1s");
- // Once upgrade finalization is stopped at the halting point, restart all
- // SCMs.
- LOG.info("Restarting all SCMs during upgrade finalization.");
- // Restarting an SCM from mini ozone actually replaces the SCM with a new
- // instance. We will use the normal upgrade finalization executor for
- // these new instances, since the last one aborted at the halting point.
- cluster.getSCMConfigurator()
- .setUpgradeFinalizationExecutor(
- new DefaultUpgradeFinalizationExecutor<>());
- List<StorageContainerManager> originalSCMs =
- cluster.getStorageContainerManagers();
+ MiniOzoneHAClusterImpl.Builder clusterBuilder = MiniOzoneCluster.newHABuilder(conf);
+ clusterBuilder.setNumOfStorageContainerManagers(NUM_SCMS)
+ .setNumOfActiveSCMs(NUM_SCMS)
+ .setSCMServiceId("scmservice")
+ .setNumOfOzoneManagers(1)
+ .setNumDatanodes(NUM_DATANODES)
+ .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
+ .setApparentVersion(HDDSVersion.SOFTWARE_VERSION.serialize())
+ .build());
- for (StorageContainerManager scm: originalSCMs) {
- cluster.restartStorageContainerManager(scm, false);
+ // Prevent terminateDatanode() from calling System.exit(1) and killing the test JVM.
+ ExitUtil.disableSystemExit();
+ LogCapturer logCapture = LogCapturer.captureLogs(RegisterEndpointTask.class);
+ // This starts the mini ozone cluster.
+ cluster = clusterBuilder.build();
+
+ // isStopped cannot be set to true unless a datanode was started first.
+ // Each datanode should be rejected since its apparent version exceeds the pre-finalized SCM's.
+ GenericTestUtils.waitFor(
+ () -> cluster.getHddsDatanodes().stream().allMatch(HddsDatanodeService::isStopped),
+ 500, 30_000);
+
+ assertThat(logCapture.getOutput()).contains("SCM rejected this datanode's registration");
+ for (StorageContainerManager scm : cluster.getStorageContainerManagersList()) {
+ assertThat(scm.getScmNodeManager().getAllNodes()).isEmpty();
}
+ }
- checkMidFinalizationConditions(haltingPoint,
- cluster.getStorageContainerManagersList());
-
- // After all SCMs were restarted, finalization should resume
- // automatically once a leader is elected.
- cluster.waitForClusterToBeReady();
-
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- // Once the leader tells the client finalization is complete, wait for all
- // followers to catch up so we can check their state.
+ @Test
+ public void testFinalization() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ init(conf, 0);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+ // Ensure all SCMs finalize, indicating the message has been propagated across them all
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
}
@Test
@@@ -167,15 -267,15 +167,15 @@@
}
// Wait for finalization from the client perspective.
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Wait for two running SCMs to finish finalization.
waitForScmsToFinalize(activeScms);
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- activeScms, 0, NUM_DATANODES);
+ activeScms, 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
// Move SCM log index farther ahead to make sure a snapshot install
// happens on the restarted SCM.
@@@ -188,16 -288,90 +188,16 @@@
}
cluster.startInactiveSCM(inactiveScm.getSCMNodeId());
- waitForScmToFinalize(inactiveScm);
+ LOG.info("Waiting for restarted SCM to finalize");
+ // When the leader sends a snapshot to the follower, it should have flushed all entries to the DB, including the
+ // apparent versin. This means the follower should see it in the DB it receives immediately to trigger finalization.
+ waitForScmToFinalize(inactiveScm, true);
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- inactiveScm, 0, NUM_DATANODES);
+ inactiveScm, 0);
// Use log to verify a snapshot was installed.
- assertThat(logCapture.getOutput()).contains("New SCM snapshot " +
- "received with metadata layout version");
- }
-
- private void waitForScmsToFinalize(Collection<StorageContainerManager> scms)
- throws Exception {
- for (StorageContainerManager scm: scms) {
- waitForScmToFinalize(scm);
- }
- }
-
- private void waitForScmToFinalize(StorageContainerManager scm)
- throws Exception {
- GenericTestUtils.waitFor(() -> !scm.isInSafeMode(), 500, 5000);
- GenericTestUtils.waitFor(() -> {
- FinalizationCheckpoint checkpoint =
- scm.getScmContext().getFinalizationCheckpoint();
- LOG.info("Waiting for SCM {} (leader? {}) to finalize. Current " +
- "finalization checkpoint is {}",
- scm.getSCMNodeId(), scm.checkLeader(), checkpoint);
- return checkpoint.hasCrossed(
- FinalizationCheckpoint.FINALIZATION_COMPLETE);
- }, 2_000, 60_000);
- }
-
- private void checkMidFinalizationConditions(
- UpgradeTestInjectionPoints haltingPoint,
- List<StorageContainerManager> scms) {
-
- // Ratis only makes sure that the Leader has processed the finalization,
- // the followers might have this in the Raft Log and not yet processed it.
- switch (haltingPoint) {
- case BEFORE_PRE_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_REQUIRED stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_REQUIRED));
- // Pipeline creation should not be frozen at this point, even on leader.
- assertTrue(scms.stream().noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_PRE_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_STARTED stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED));
- // Pipeline creation should be frozen on nodes where the finalization checkpoint is FINALIZATION_STARTED,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED)
- .allMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_COMPLETE_FINALIZATION:
- // At least one node (leader) should be in the MLV_EQUALS_SLV stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV));
- // Pipeline creation should not be frozen on nodes where the finalization checkpoint is MLV_EQUALS_SLV,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV)
- .noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_POST_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_COMPLETE stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE));
- // Pipeline creation should not be frozen on nodes where the finalization checkpoint is FINALIZATION_COMPLETE,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE)
- .noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- default:
- fail("Unknown halting point in test: " + haltingPoint);
- }
+ assertThat(logCapture.getOutput()).contains("New snapshot received with higher apparent version " +
+ HDDSVersion.SOFTWARE_VERSION + ". Attempting to finalize to that version.");
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
index 482662c9df8,3c1d3ed6faf..0c3bc3b37bc
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
@@@ -41,9 -45,11 +41,9 @@@ import org.apache.hadoop.hdds.conf.Ozon
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMPerformanceMetrics;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
- import org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils;
-import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
+ import org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.UniformDatanodesFactory;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@@ -136,8 -151,18 +136,8 @@@ public class TestBlockDeletionService
GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
// UPGRADE SCM (if specified)
- // Step 5: wait for finalizing upgrade
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(() -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- fail("finalization client failed", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// POST-UPGRADE
//Step 6: Repeat the same steps in pre-upgrade
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
index f6e0baec5e5,44131d81c88..aea70b1d253
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
@@@ -51,8 -51,11 +51,9 @@@ import static org.apache.hadoop.ozone.s
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
import static org.apache.ozone.test.LambdaTestUtils.await;
+ import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
index 5bfbe349e83,884c5fa310b..f913b7a79ad
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
@@@ -111,12 -111,12 +111,12 @@@ public abstract class OMClientRequest i
*/
public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
- LayoutVersion layoutVersion = LayoutVersion.newBuilder()
- .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion())
+ LayoutVersion apparentVersion = LayoutVersion.newBuilder()
+ .setVersion(ozoneManager.getVersionManager().getApparentVersion().serialize())
.build();
omRequest = getOmRequest().toBuilder()
- .setUserInfo(getUserIfNotExists(ozoneManager))
- .setLayoutVersion(layoutVersion).build();
+ .setLayoutVersion(apparentVersion)
- .setUserInfo(getUserInfo()).build();
++ .setUserInfo(getUserIfNotExists(ozoneManager)).build();
return omRequest;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org