You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by er...@apache.org on 2026/07/16 20:57:36 UTC
(ozone) branch HDDS-14496-zdu updated (98c55fe4905 -> e16d646d29b)
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a change to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 98c55fe4905 HDDS-15848. Add new upgrade action annotations for component versions (#10744)
add b17e90e100a HDDS-15708. List ignored dependencies for dependabot (#10676)
add 3985622f6f6 HDDS-15818. Bump common-custom-user-data-maven-extension to 2.3.0 (#10715)
add 3a5861acb6f HDDS-15821. Bump iceberg to 1.10.2 (#10719)
add bc74b52fbca HDDS-15807. Add javadoc for SCM Ratis server placeholder TODOs (#10706)
add 4575cf9f3f4 HDDS-15827. Wait for SCM pipelines in TestDnRatisLogParser (#10725)
add 78cb4936daa HDDS-15822. Fix flaky TestStorageVolumeChecker#testNumScansSkipped (#10720)
add 5e18bfb916f HDDS-15825. Bump jnr-posix to 3.2.1, jnr-constants to 0.11.0 (#10716)
add 5b803eb115f HDDS-15830. Bump jline to 3.30.15 (#10729)
add db0413928ac HDDS-15820. Bump awssdk to 2.46.21 (#10718)
add 991798c5925 HDDS-15804. Use Set instead of List for ACLs in OMKeyRequest (#10727)
add 7440e9fc278 HDDS-15806. Replace sleep with waitFor in scm.node tests (#10704)
add 2c7fac752ab HDDS-15832. Wait for healthy KDC before starting S3G (#10730)
add 2684ab4c2a8 HDDS-15819. Bump log4j2 to 2.26.1 (#10717)
add 9d0c914064c HDDS-15834. Bump cyclonedx-maven-plugin to 2.9.2 (#10734)
add 04958bbe961 HDDS-15831. CompleteMultipartUpload with no parts should return MalformedXML (#10731)
add 8f7bf38bce4 HDDS-15730. Support more StatefulService types in DBScanner (#10654)
add c35148d8561 HDDS-15793. Bump ESLint to 9.39.4 (#10697)
add b7478f1f4f9 HDDS-15786. Intermittent pipeline limit exceeded in debug tests (#10738)
add 9de98bd4202 HDDS-15726. Improve SCM block deletion summary error handling (#10650)
add 1b9491ae33e HDDS-15842. Speed up TestDuplicateContainerDirScannerIntegration (#10740)
add 2c48d33f0c8 HDDS-9154. Design doc for snapshot diff optimization. (#10335)
add 89a45ea68e0 HDDS-15751. Rename test parent classes for naming convention (#10713)
add 958465f6e28 HDDS-13472. Shut down datanode if Error occurs while processing delete blocks command (#10733)
add 87d2bd8577b HDDS-15387. Selectively deserialize KeyInfo/DirectoryInfo fields for snapshot diff (#10603)
add ff5d03288c4 HDDS-15084. Core Runtime: SCM + OM (#10456)
add 06d972484dc HDDS-15843. Update upgrade/xcompat tests to use 2.1.1 (#10739)
add bf142be6116 HDDS-14661. Handle split table writes for Multipart Upload (#10588)
add 7d89300dcf7 HDDS-13063. Support splitting json output to multiple valid json files in a directory (#10604)
add 7a49f649958 HDDS-15794. StreamBlockInputStream.onNext may throw IndexOutOfBoundsException on short payloads (#10698)
add 8e32bc46964 HDDS-14957. Separate OpenKeyInfo to differentiate KeyInfo (#10181)
add 4bfdb1f7546 HDDS-14957. (addendum) Fix compile error in TestSnapshotDiffValueParser
add e09eeaa7285 HDDS-15616. ListBuckets API ignores MaxBuckets parameter and returns all buckets (#10639)
add cb4d088e223 Revert "HDDS-15467. Do not fall back to the OM starter user in OMClientRequest" (#10756)
add f9e7bc4cffc HDDS-15836. Add Archiver.appendFile for incremental TAR writes (#10735)
add d2f92864686 HDDS-15851. Return 403 for expired or out-of-range pre-signed S3 URLs (#10749)
add 575fe4f1cd7 HDDS-15837. Add health-aware container ID listing in SCM (#10736)
add a8fcb09c023 HDDS-15823. Fix flaky TestFailureHandlingByClient#testContainerExclusionWithClosedContainerException (#10721)
add e31f36d2316 HDDS-15667. `containerbalancer status`: show stop reason and iteration details after stop (#10681)
add 418d07dde59 HDDS-15656. Fix non-atomic data size accumulation in ContainerBalancer move callback (#10708)
add b18ea9284c4 HDDS-15600. Fix ListObjects response for encoding-type and empty delimiter (#10586).
add d4fed1c90ea HDDS-15812. Rename test utils for naming convention (#10766)
add 36b90e5dde8 HDDS-15456. Add SCM DB lookup and identify orphan(wrt SCM) and deleted-but-present containers (#10547)
add 9d0339cbfaf HDDS-14989. Delay follower SCM DN server start until Ratis log catch-up. (#10617).
add e0bdbc2e3f9 HDDS-15765. Fail fast when --node-id is omitted for OM compact/defrag on an HA cluster (#10699)
add a218b3bfea3 HDDS-15849. Force channel shutdown in XceiverClientGrpc.close (#10747)
add 398709d6b68 HDDS-15709. Replace random name suffix with uniqueObjectName (#10773)
add 6fd35fbf92f HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs (#10209)
add 4a819cc4eee HDDS-15171. Add available space check on follower during bootstrap. (#10185)
add d2700a4b01b HDDS-15678. OFS isDirectory/isFile should not trigger pipeline refresh or return block locations (#10741)
add 1a9439452d9 HDDS-15809. Validate x-amz-copy-source-range on UploadPartCopy (#10711).
add 6ba0d57bb4d HDDS-15867. [JDK25] Bump byte-buddy to 1.18.11 (#10776)
add b981afbf89e HDDS-15508. Add 2.2.0 to compatibility/upgrade test (#10782)
add 6ac2289aafd HDDS-15876. Remove ozone-csi (#10780)
new e16d646d29b Merge branch 'master' into HDDS-14496-zdu-master-3
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/dependabot.yml | 56 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
.run/CsiServer.run.xml | 33 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 40 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 9 +-
.../TestXceiverClientManagerCloseContention.java | 161 +++
.../scm/storage/TestStreamBlockInputStream.java | 156 +++
hadoop-hdds/common/src/main/conf/ozone-env.sh | 11 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 1 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +
.../common/src/main/resources/ozone-default.xml | 23 +
.../commandhandler/DeleteBlocksCommandHandler.java | 7 +
.../TestDeleteBlocksCommandHandler.java | 108 ++
...achine.java => ContainerStateMachineTests.java} | 4 +-
.../ratis/TestContainerStateMachineFollower.java | 2 +-
.../ratis/TestContainerStateMachineLeader.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 16 +-
...ersAbstract.java => ContainerScannerTests.java} | 2 +-
.../TestBackgroundContainerDataScanner.java | 2 +-
.../TestBackgroundContainerMetadataScanner.java | 2 +-
.../ozoneimpl/TestOnDemandContainerScanner.java | 2 +-
.../docs/content/design/efficient-snapdiff.md | 254 ++++
hadoop-hdds/docs/content/interface/CSI.md | 93 --
hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes
hadoop-hdds/docs/content/interface/CSI.zh.md | 92 --
hadoop-hdds/docs/content/tools/Admin.md | 4 +-
hadoop-hdds/docs/content/tools/Repair.md | 4 +-
.../{TestCoderBase.java => CoderTests.java} | 2 +-
...estRSRawCoderBase.java => RSRawCoderTests.java} | 4 +-
.../{TestRawCoderBase.java => RawCoderTests.java} | 6 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 2 +-
.../rawcoder/TestNativeXORRawCoder.java | 2 +-
.../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 2 +-
...tXORRawCoderBase.java => XORRawCoderTests.java} | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 76 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +-
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../interface-admin/src/main/resources/proto.lock | 18 +
.../utils/{TestUtils.java => RocksTestUtils.java} | 4 +-
.../utils/db/TestManagedRawSSTFileIterator.java | 4 +-
.../hadoop/hdds/utils/db/TestSstFileSetReader.java | 6 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 9 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 +-
.../SCMDeletedBlockTransactionStatusManager.java | 133 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 4 +-
.../hdds/scm/container/ContainerManagerImpl.java | 5 +-
.../hdds/scm/container/ContainerStateManager.java | 5 +-
.../scm/container/ContainerStateManagerImpl.java | 5 +-
.../scm/container/balancer/ContainerBalancer.java | 28 +-
.../balancer/ContainerBalancerStatusInfo.java | 43 +-
.../balancer/ContainerBalancerStopReason.java | 63 +
.../container/balancer/ContainerBalancerTask.java | 143 +-
.../scm/container/states/ContainerStateMap.java | 49 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 5 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 135 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +-
.../hdds/scm/ha/StatefulServiceDefinition.java | 48 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 83 +-
.../hdds/scm/security/RootCARotationManager.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +-
.../scm/container/TestContainerStateManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 43 +
.../balancer/TestContainerBalancerStatusInfo.java | 66 +
.../balancer/TestContainerBalancerTask.java | 58 +
...andler.java => MisReplicationHandlerTests.java} | 2 +-
.../replication/TestECMisReplicationHandler.java | 2 +-
.../TestRatisMisReplicationHandler.java | 2 +-
.../container/states/TestContainerStateMap.java | 102 +-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +-
.../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 5 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +-
.../ozone/admin/om/snapshot/DefragSubCommand.java | 11 +-
.../datanode/TestContainerBalancerSubCommand.java | 242 +++-
.../admin/om/snapshot/TestDefragSubCommand.java | 22 +
.../container/analyze/AnalyzeSubcommand.java | 234 +++-
.../analyze/ScmContainerMetadataReader.java | 155 ++
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 51 +-
.../ozone/debug/replicas/ReplicasVerify.java | 112 +-
.../analyze/ContainerAnalyzeTestHelper.java | 37 +
.../container/analyze/TestAnalyzeSubcommand.java | 188 ++-
.../analyze/TestScmContainerMetadataReader.java | 100 ++
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 11 +-
.../hadoop/ozone/repair/om/TestCompactOMDB.java | 12 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../ozone/client/protocol/ClientProtocol.java | 22 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
.../hadoop/ozone/client/TestOzoneBucket.java | 72 +
.../client/TestRpcClientGetFileStatusHeadOp.java | 93 ++
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 22 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 +
.../hadoop/ozone/om/helpers/AclListBuilder.java | 7 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 74 +-
.../ozone/om/helpers/OmMultipartAbortInfo.java | 42 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 17 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 22 +
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 55 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../ozone/security/acl/IAccessAuthorizer.java | 16 +-
.../csi/dev-support/findbugsExcludeFile.xml | 22 -
hadoop-ozone/csi/pom.xml | 290 ----
.../apache/hadoop/ozone/csi/ControllerService.java | 116 --
.../org/apache/hadoop/ozone/csi/CsiServer.java | 177 ---
.../apache/hadoop/ozone/csi/IdentityService.java | 72 -
.../org/apache/hadoop/ozone/csi/NodeService.java | 148 --
.../org/apache/hadoop/ozone/csi/package-info.java | 21 -
hadoop-ozone/csi/src/main/proto/csi.proto | 1323 -----------------
hadoop-ozone/csi/src/main/resources/proto.lock | 1479 --------------------
.../dev-support/intellij/ozone-site-ha.xml | 10 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
hadoop-ozone/dist/pom.xml | 5 -
.../dist/src/main/compose/common/replicas-test.sh | 2 +
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 20 -
.../src/main/compose/ozone-csi/docker-compose.yaml | 64 -
.../dist/src/main/compose/ozone-csi/docker-config | 44 -
.../dist/src/main/compose/ozone-csi/test.sh | 31 -
.../main/compose/ozonesecure-ha/s3-haproxy.yaml | 3 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 11 +-
.../dist/src/main/compose/xcompat/clients.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 53 -
.../main/k8s/definitions/ozone-csi/csi-crd.yaml | 21 -
.../main/k8s/definitions/ozone-csi/csi-node.yaml | 95 --
.../main/k8s/definitions/ozone-csi/csi-rbac.yaml | 66 -
.../definitions/ozone-csi/csi-storageclass.yaml | 20 -
.../k8s/definitions/ozone-csi/definitions/csi.yaml | 28 -
.../k8s/definitions/test-webserver/flekszible.yaml | 16 -
.../test-webserver/webserver-deployment.yaml | 50 -
.../test-webserver/webserver-service.yaml | 29 -
.../test-webserver/webserver-volume.yaml | 29 -
.../src/main/k8s/examples/ozone-dev/Flekszible | 7 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 6 +-
.../examples/ozone-dev/csi/csi-node-daemonset.yaml | 97 --
.../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 --
.../csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone-dev/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../examples/ozone-dev/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
.../dist/src/main/k8s/examples/ozone/Flekszible | 5 -
.../main/k8s/examples/ozone/config-configmap.yaml | 6 +-
.../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 97 --
.../examples/ozone/csi/csi-ozone-clusterrole.yaml | 98 --
.../ozone/csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../k8s/examples/ozone/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 2 -
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
hadoop-ozone/dist/src/main/smoketest/csi.robot | 34 -
.../src/main/smoketest/s3/MultipartUpload.robot | 15 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 7 -
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../AbstractTestStorageDistributionEndpoint.java | 14 +-
...ndpointUtil.java => ReconEndpointTestUtil.java} | 6 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 36 +-
hadoop-ozone/integration-test-s3/pom.xml | 7 +-
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 169 ++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 239 +++-
hadoop-ozone/integration-test/pom.xml | 5 -
...eldsBase.java => ConfigurationFieldsTests.java} | 8 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 44 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 89 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 +-
.../fs/ozone/TestOFSIsDirectoryBenchmark.java | 153 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 6 +-
.../hdds/scm/TestContainerReportWithKeys.java | 7 +-
.../TestSCMFollowerCatchupWithContainerReport.java | 336 +++++
.../TestReplicationManagerIntegration.java | 8 +-
.../node/{TestNodeUtil.java => NodeTestUtil.java} | 4 +-
.../scm/node/TestDecommissionAndMaintenance.java | 16 +-
.../hdds/scm/safemode/TestSafeModeSCMHA.java | 18 +-
...UpgradeUtils.java => HddsUpgradeTestUtils.java} | 6 +-
.../TestDNDataDistributionFinalization.java | 4 +-
.../TestScmDataDistributionFinalization.java | 14 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 18 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 9 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 25 +-
.../client/rpc/TestFailureHandlingByClient.java | 13 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 6 +-
...tInputStreamBase.java => InputStreamTests.java} | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 12 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 30 +-
.../rpc/read/TestStreamBlockInputStream.java | 12 +-
.../ozone/client/rpc/read/TestStreamRead.java | 6 +-
.../rpc/read/TestStreamReadDatanodeFailover.java | 4 +-
...estDuplicateContainerDirScannerIntegration.java | 1 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 4 +
....java => ContainerScannerIntegrationTests.java} | 2 +-
...tBackgroundContainerDataScannerIntegration.java | 2 +-
...kgroundContainerMetadataScannerIntegration.java | 2 +-
.../TestOnDemandContainerScannerIntegration.java | 2 +-
...estDataValidate.java => DataValidateTests.java} | 2 +-
.../freon/TestDataValidateWithDummyContainers.java | 2 +-
.../TestDataValidateWithSafeByteOperations.java | 2 +-
.../TestDataValidateWithUnsafeByteOperations.java | 2 +-
.../ozone/local/TestLocalOzoneClusterRuntime.java | 136 ++
.../ozone/om/AbstractOzoneManagerHATest.java | 11 +-
.../om/{TestBucket.java => BucketForTesting.java} | 16 +-
...d.java => OzoneManagerHAFollowerReadTests.java} | 2 +-
...zoneManagerHA.java => OzoneManagerHATests.java} | 2 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 8 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 12 +-
.../apache/hadoop/ozone/om/TestOMBootstrap.java | 5 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 80 +-
.../ozone/om/TestOMRatisSnapshotTransfer.java | 5 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 13 +-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 9 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 15 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 11 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 8 +-
.../ozone/om/service/TestBlockDeletionService.java | 4 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 6 +-
.../{TestOmSnapshot.java => OmSnapshotTests.java} | 47 +-
.../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 +
.../ozone/om/snapshot/TestOmSnapshotDisabled.java | 8 +-
.../om/snapshot/TestOmSnapshotDisabledRestart.java | 8 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 +-
.../TestOmSnapshotFsoWithoutNativeLib.java | 2 +-
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 2 +-
.../om/snapshot/TestOmSnapshotObjectStore.java | 2 +-
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 5 +-
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +-
...skBalancerDuringDecommissionAndMaintenance.java | 4 +-
.../ozone/shell/TestOzoneDebugReplicasVerify.java | 136 ++
.../src/main/proto/OmClientProtocol.proto | 4 +
.../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 +-
.../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 36 +
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 65 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 26 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 23 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 +++-
.../hadoop/ozone/om/request/OMClientRequest.java | 39 +-
.../om/request/key/OMAllocateBlockRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 3 -
.../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 42 +-
.../multipart/S3MultipartUploadAbortRequest.java | 42 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 9 +-
.../S3MultipartUploadCommitPartRequest.java | 74 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 8 +-
.../S3MultipartUploadCompleteRequest.java | 78 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 +-
.../om/request/util/OMMultipartUploadUtils.java | 102 ++
.../AbstractS3MultipartAbortResponse.java | 53 +-
.../S3ExpiredMultipartUploadsAbortResponse.java | 3 +-
.../multipart/S3MultipartUploadAbortResponse.java | 30 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 14 +-
.../S3MultipartUploadCommitPartResponse.java | 23 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 8 +-
.../S3MultipartUploadCompleteResponse.java | 15 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 +-
.../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +-
.../ozone/om/snapshot/SnapshotDiffValueParser.java | 359 +++++
.../protocolPB/OzoneManagerRequestHandler.java | 13 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 33 +
.../TestOmRatisSnapshotProvider.java | 59 +
.../TestOMClientRequestUserInfoFallback.java | 106 --
.../om/request/key/TestOMKeyCommitRequest.java | 2 -
.../TestS3MultipartUploadCommitPartRequest.java | 355 +++++
.../s3/multipart/S3MultipartResponseTests.java | 10 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 4 +-
.../om/snapshot/TestSnapshotDiffValueParser.java | 249 ++++
.../protocolPB/TestOzoneManagerRequestHandler.java | 83 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 86 ++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 67 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 11 +
.../TestBasicRootedOzoneClientAdapterHeadOp.java | 176 +++
.../fs/ozone/TestRootedOzoneFileSystemHeadOp.java | 174 +++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
hadoop-ozone/pom.xml | 1 -
.../webapps/recon/ozone-recon-web/.eslintignore | 26 -
.../recon/ozone-recon-web/eslint.config.mjs | 109 ++
.../webapps/recon/ozone-recon-web/package.json | 15 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1111 +++++++--------
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +-
.../ozone/s3/commontypes/RequestParameters.java | 11 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 7 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 5 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 13 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 37 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 7 +-
...ser.java => AccessDeniedResourceException.java} | 12 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 +
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 4 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 17 +
.../hadoop/ozone/s3/endpoint/TestRootList.java | 94 ++
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 12 +-
.../s3/signature/TestAWSSignatureProcessor.java | 37 +
.../signature/TestAuthorizationV4QueryParser.java | 16 +-
hadoop-ozone/tools/pom.xml | 12 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 294 +++-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 9 +-
pom.xml | 32 +-
342 files changed, 9510 insertions(+), 7440 deletions(-)
delete mode 100644 .run/CsiServer.run.xml
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerCloseContention.java
rename hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/{TestContainerStateMachine.java => ContainerStateMachineTests.java} (99%)
rename hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/{TestContainerScannersAbstract.java => ContainerScannerTests.java} (99%)
create mode 100644 hadoop-hdds/docs/content/design/efficient-snapdiff.md
delete mode 100644 hadoop-hdds/docs/content/interface/CSI.md
delete mode 100644 hadoop-hdds/docs/content/interface/CSI.png
delete mode 100644 hadoop-hdds/docs/content/interface/CSI.zh.md
rename hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/{TestCoderBase.java => CoderTests.java} (99%)
rename hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/{TestRSRawCoderBase.java => RSRawCoderTests.java} (97%)
rename hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/{TestRawCoderBase.java => RawCoderTests.java} (98%)
rename hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/{TestXORRawCoderBase.java => XORRawCoderTests.java} (95%)
rename hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/{TestUtils.java => RocksTestUtils.java} (96%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStopReason.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/RadixNode.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceDefinition.java (54%)
rename hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/{TestMisReplicationHandler.java => MisReplicationHandlerTests.java} (99%)
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/datanode/container/analyze/ScmContainerMetadataReader.java
create mode 100644 hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/datanode/container/analyze/TestScmContainerMetadataReader.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneBucket.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestRpcClientGetFileStatusHeadOp.java
delete mode 100644 hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml
delete mode 100644 hadoop-ozone/csi/pom.xml
delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java
delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java
delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java
delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java
delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/package-info.java
delete mode 100644 hadoop-ozone/csi/src/main/proto/csi.proto
delete mode 100644 hadoop-ozone/csi/src/main/resources/proto.lock
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/.env
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
delete mode 100755 hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-controller.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-crd.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-node.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-rbac.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-storageclass.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/definitions/csi.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/flekszible.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-deployment.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-service.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-volume.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-node-daemonset.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrole.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrolebinding.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-serviceaccount.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-provisioner-deployment.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/org.apache.hadoop.ozone-csidriver.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/ozone-storageclass.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-deployment.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-service.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-node-daemonset.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrole.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrolebinding.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-serviceaccount.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-provisioner-deployment.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/org.apache.hadoop.ozone-csidriver.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/ozone-storageclass.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-deployment.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml
delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-service.yaml
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/csi.robot
rename hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/{TestReconEndpointUtil.java => ReconEndpointTestUtil.java} (97%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/conf/{TestConfigurationFieldsBase.java => ConfigurationFieldsTests.java} (99%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOFSIsDirectoryBenchmark.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMFollowerCatchupWithContainerReport.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/{TestNodeUtil.java => NodeTestUtil.java} (98%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/{TestHddsUpgradeUtils.java => HddsUpgradeTestUtils.java} (97%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/{TestInputStreamBase.java => InputStreamTests.java} (99%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/{TestContainerScannerIntegrationAbstract.java => ContainerScannerIntegrationTests.java} (99%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/{TestDataValidate.java => DataValidateTests.java} (99%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterRuntime.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{TestBucket.java => BucketForTesting.java} (90%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{TestOzoneManagerHAFollowerRead.java => OzoneManagerHAFollowerReadTests.java} (96%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{TestOzoneManagerHA.java => OzoneManagerHATests.java} (94%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshot.java => OmSnapshotTests.java} (98%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestUserInfoFallback.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java
create mode 100644 hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java
create mode 100644 hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemHeadOp.java
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/.eslintignore
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/eslint.config.mjs
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/{SignatureParser.java => AccessDeniedResourceException.java} (73%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge branch 'master' into HDDS-14496-zdu-master-3
Posted by er...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e16d646d29bda26bd5da9d98ba9c2e828348dfac
Merge: 98c55fe4905 6ac2289aafd
Author: Ethan Rose <er...@apache.org>
AuthorDate: Thu Jul 16 16:51:04 2026 -0400
Merge branch 'master' into HDDS-14496-zdu-master-3
* master: (53 commits)
HDDS-15876. Remove ozone-csi (#10780)
HDDS-15508. Add 2.2.0 to compatibility/upgrade test (#10782)
HDDS-15867. [JDK25] Bump byte-buddy to 1.18.11 (#10776)
HDDS-15809. Validate x-amz-copy-source-range on UploadPartCopy (#10711).
HDDS-15678. OFS isDirectory/isFile should not trigger pipeline refresh or return block locations (#10741)
HDDS-15171. Add available space check on follower during bootstrap. (#10185)
HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs (#10209)
HDDS-15709. Replace random name suffix with uniqueObjectName (#10773)
HDDS-15849. Force channel shutdown in XceiverClientGrpc.close (#10747)
HDDS-15765. Fail fast when --node-id is omitted for OM compact/defrag on an HA cluster (#10699)
HDDS-14989. Delay follower SCM DN server start until Ratis log catch-up. (#10617).
HDDS-15456. Add SCM DB lookup and identify orphan(wrt SCM) and deleted-but-present containers (#10547)
HDDS-15812. Rename test utils for naming convention (#10766)
HDDS-15600. Fix ListObjects response for encoding-type and empty delimiter (#10586).
HDDS-15656. Fix non-atomic data size accumulation in ContainerBalancer move callback (#10708)
HDDS-15667. `containerbalancer status`: show stop reason and iteration details after stop (#10681)
HDDS-15823. Fix flaky TestFailureHandlingByClient#testContainerExclusionWithClosedContainerException (#10721)
HDDS-15837. Add health-aware container ID listing in SCM (#10736)
HDDS-15851. Return 403 for expired or out-of-range pre-signed S3 URLs (#10749)
HDDS-15836. Add Archiver.appendFile for incremental TAR writes (#10735)
...
Conflicts:
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
hadoop-ozone/dist/src/main/compose/upgrade/test.sh
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
.github/dependabot.yml | 56 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
.run/CsiServer.run.xml | 33 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 40 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 9 +-
.../TestXceiverClientManagerCloseContention.java | 161 +++
.../scm/storage/TestStreamBlockInputStream.java | 156 +++
hadoop-hdds/common/src/main/conf/ozone-env.sh | 11 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 1 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +
.../common/src/main/resources/ozone-default.xml | 23 +
.../commandhandler/DeleteBlocksCommandHandler.java | 7 +
.../TestDeleteBlocksCommandHandler.java | 108 ++
...achine.java => ContainerStateMachineTests.java} | 4 +-
.../ratis/TestContainerStateMachineFollower.java | 2 +-
.../ratis/TestContainerStateMachineLeader.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 16 +-
...ersAbstract.java => ContainerScannerTests.java} | 2 +-
.../TestBackgroundContainerDataScanner.java | 2 +-
.../TestBackgroundContainerMetadataScanner.java | 2 +-
.../ozoneimpl/TestOnDemandContainerScanner.java | 2 +-
.../docs/content/design/efficient-snapdiff.md | 254 ++++
hadoop-hdds/docs/content/interface/CSI.md | 93 --
hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes
hadoop-hdds/docs/content/interface/CSI.zh.md | 92 --
hadoop-hdds/docs/content/tools/Admin.md | 4 +-
hadoop-hdds/docs/content/tools/Repair.md | 4 +-
.../{TestCoderBase.java => CoderTests.java} | 2 +-
...estRSRawCoderBase.java => RSRawCoderTests.java} | 4 +-
.../{TestRawCoderBase.java => RawCoderTests.java} | 6 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 2 +-
.../rawcoder/TestNativeXORRawCoder.java | 2 +-
.../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 2 +-
...tXORRawCoderBase.java => XORRawCoderTests.java} | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 76 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +-
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../interface-admin/src/main/resources/proto.lock | 18 +
.../utils/{TestUtils.java => RocksTestUtils.java} | 4 +-
.../utils/db/TestManagedRawSSTFileIterator.java | 4 +-
.../hadoop/hdds/utils/db/TestSstFileSetReader.java | 6 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 9 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 +-
.../SCMDeletedBlockTransactionStatusManager.java | 133 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 4 +-
.../hdds/scm/container/ContainerManagerImpl.java | 5 +-
.../hdds/scm/container/ContainerStateManager.java | 5 +-
.../scm/container/ContainerStateManagerImpl.java | 5 +-
.../scm/container/balancer/ContainerBalancer.java | 28 +-
.../balancer/ContainerBalancerStatusInfo.java | 43 +-
.../balancer/ContainerBalancerStopReason.java | 63 +
.../container/balancer/ContainerBalancerTask.java | 143 +-
.../scm/container/states/ContainerStateMap.java | 49 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 5 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 135 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +-
.../hdds/scm/ha/StatefulServiceDefinition.java} | 38 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 83 +-
.../hdds/scm/security/RootCARotationManager.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +-
.../scm/container/TestContainerStateManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 43 +
.../balancer/TestContainerBalancerStatusInfo.java | 66 +
.../balancer/TestContainerBalancerTask.java | 58 +
...andler.java => MisReplicationHandlerTests.java} | 2 +-
.../replication/TestECMisReplicationHandler.java | 2 +-
.../TestRatisMisReplicationHandler.java | 2 +-
.../container/states/TestContainerStateMap.java | 102 +-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +-
.../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 5 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +-
.../ozone/admin/om/snapshot/DefragSubCommand.java | 11 +-
.../datanode/TestContainerBalancerSubCommand.java | 242 +++-
.../admin/om/snapshot/TestDefragSubCommand.java | 22 +
.../container/analyze/AnalyzeSubcommand.java | 234 +++-
.../analyze/ScmContainerMetadataReader.java | 155 ++
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 51 +-
.../ozone/debug/replicas/ReplicasVerify.java | 112 +-
.../analyze/ContainerAnalyzeTestHelper.java | 37 +
.../container/analyze/TestAnalyzeSubcommand.java | 188 ++-
.../analyze/TestScmContainerMetadataReader.java | 100 ++
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 11 +-
.../hadoop/ozone/repair/om/TestCompactOMDB.java | 12 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../ozone/client/protocol/ClientProtocol.java | 22 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
.../hadoop/ozone/client/TestOzoneBucket.java | 72 +
.../client/TestRpcClientGetFileStatusHeadOp.java | 93 ++
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 22 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 +
.../hadoop/ozone/om/helpers/AclListBuilder.java | 7 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 74 +-
.../ozone/om/helpers/OmMultipartAbortInfo.java | 42 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 17 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 22 +
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 55 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../ozone/security/acl/IAccessAuthorizer.java | 16 +-
.../csi/dev-support/findbugsExcludeFile.xml | 22 -
hadoop-ozone/csi/pom.xml | 290 ----
.../apache/hadoop/ozone/csi/ControllerService.java | 116 --
.../org/apache/hadoop/ozone/csi/CsiServer.java | 177 ---
.../apache/hadoop/ozone/csi/IdentityService.java | 72 -
.../org/apache/hadoop/ozone/csi/NodeService.java | 148 --
.../org/apache/hadoop/ozone/csi/package-info.java | 21 -
hadoop-ozone/csi/src/main/proto/csi.proto | 1323 -----------------
hadoop-ozone/csi/src/main/resources/proto.lock | 1479 --------------------
.../dev-support/intellij/ozone-site-ha.xml | 10 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
hadoop-ozone/dist/pom.xml | 5 -
.../dist/src/main/compose/common/replicas-test.sh | 2 +
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 20 -
.../src/main/compose/ozone-csi/docker-compose.yaml | 64 -
.../dist/src/main/compose/ozone-csi/docker-config | 44 -
.../dist/src/main/compose/ozone-csi/test.sh | 31 -
.../main/compose/ozonesecure-ha/s3-haproxy.yaml | 3 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 11 +-
.../dist/src/main/compose/xcompat/clients.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 53 -
.../main/k8s/definitions/ozone-csi/csi-crd.yaml | 21 -
.../main/k8s/definitions/ozone-csi/csi-node.yaml | 95 --
.../main/k8s/definitions/ozone-csi/csi-rbac.yaml | 66 -
.../definitions/ozone-csi/csi-storageclass.yaml | 20 -
.../k8s/definitions/ozone-csi/definitions/csi.yaml | 28 -
.../k8s/definitions/test-webserver/flekszible.yaml | 16 -
.../test-webserver/webserver-deployment.yaml | 50 -
.../test-webserver/webserver-service.yaml | 29 -
.../test-webserver/webserver-volume.yaml | 29 -
.../src/main/k8s/examples/ozone-dev/Flekszible | 7 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 6 +-
.../examples/ozone-dev/csi/csi-node-daemonset.yaml | 97 --
.../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 --
.../csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone-dev/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../examples/ozone-dev/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
.../dist/src/main/k8s/examples/ozone/Flekszible | 5 -
.../main/k8s/examples/ozone/config-configmap.yaml | 6 +-
.../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 97 --
.../examples/ozone/csi/csi-ozone-clusterrole.yaml | 98 --
.../ozone/csi/csi-ozone-clusterrolebinding.yaml | 28 -
.../ozone/csi/csi-ozone-serviceaccount.yaml | 21 -
.../ozone/csi/csi-provisioner-deployment.yaml | 54 -
.../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 -
.../k8s/examples/ozone/csi/ozone-storageclass.yaml | 21 -
.../ozone-csi-test-webserver-deployment.yaml | 50 -
...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 -
.../pv-test/ozone-csi-test-webserver-service.yaml | 29 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 2 -
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
hadoop-ozone/dist/src/main/smoketest/csi.robot | 34 -
.../src/main/smoketest/s3/MultipartUpload.robot | 15 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 7 -
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../AbstractTestStorageDistributionEndpoint.java | 14 +-
...ndpointUtil.java => ReconEndpointTestUtil.java} | 6 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 36 +-
hadoop-ozone/integration-test-s3/pom.xml | 7 +-
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 169 ++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 239 +++-
hadoop-ozone/integration-test/pom.xml | 5 -
...eldsBase.java => ConfigurationFieldsTests.java} | 8 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 44 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 89 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 +-
.../fs/ozone/TestOFSIsDirectoryBenchmark.java | 153 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 6 +-
.../hdds/scm/TestContainerReportWithKeys.java | 7 +-
.../TestSCMFollowerCatchupWithContainerReport.java | 336 +++++
.../TestReplicationManagerIntegration.java | 8 +-
.../node/{TestNodeUtil.java => NodeTestUtil.java} | 4 +-
.../scm/node/TestDecommissionAndMaintenance.java | 16 +-
.../hdds/scm/safemode/TestSafeModeSCMHA.java | 18 +-
...UpgradeUtils.java => HddsUpgradeTestUtils.java} | 6 +-
.../TestDNDataDistributionFinalization.java | 4 +-
.../TestScmDataDistributionFinalization.java | 14 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 18 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 9 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 25 +-
.../client/rpc/TestFailureHandlingByClient.java | 13 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 6 +-
...tInputStreamBase.java => InputStreamTests.java} | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 12 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 30 +-
.../rpc/read/TestStreamBlockInputStream.java | 12 +-
.../ozone/client/rpc/read/TestStreamRead.java | 6 +-
.../rpc/read/TestStreamReadDatanodeFailover.java | 4 +-
...estDuplicateContainerDirScannerIntegration.java | 1 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 4 +
....java => ContainerScannerIntegrationTests.java} | 2 +-
...tBackgroundContainerDataScannerIntegration.java | 2 +-
...kgroundContainerMetadataScannerIntegration.java | 2 +-
.../TestOnDemandContainerScannerIntegration.java | 2 +-
...estDataValidate.java => DataValidateTests.java} | 2 +-
.../freon/TestDataValidateWithDummyContainers.java | 2 +-
.../TestDataValidateWithSafeByteOperations.java | 2 +-
.../TestDataValidateWithUnsafeByteOperations.java | 2 +-
.../ozone/local/TestLocalOzoneClusterRuntime.java | 136 ++
.../ozone/om/AbstractOzoneManagerHATest.java | 11 +-
.../om/{TestBucket.java => BucketForTesting.java} | 16 +-
...d.java => OzoneManagerHAFollowerReadTests.java} | 2 +-
...zoneManagerHA.java => OzoneManagerHATests.java} | 2 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 8 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 12 +-
.../apache/hadoop/ozone/om/TestOMBootstrap.java | 5 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 80 +-
.../ozone/om/TestOMRatisSnapshotTransfer.java | 5 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 13 +-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 9 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 15 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 11 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 8 +-
.../ozone/om/service/TestBlockDeletionService.java | 4 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 6 +-
.../{TestOmSnapshot.java => OmSnapshotTests.java} | 47 +-
.../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 +
.../ozone/om/snapshot/TestOmSnapshotDisabled.java | 8 +-
.../om/snapshot/TestOmSnapshotDisabledRestart.java | 8 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 +-
.../TestOmSnapshotFsoWithoutNativeLib.java | 2 +-
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 2 +-
.../om/snapshot/TestOmSnapshotObjectStore.java | 2 +-
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 5 +-
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +-
...skBalancerDuringDecommissionAndMaintenance.java | 4 +-
.../ozone/shell/TestOzoneDebugReplicasVerify.java | 136 ++
.../src/main/proto/OmClientProtocol.proto | 4 +
.../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 +-
.../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 36 +
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 65 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 26 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 23 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 +++-
.../hadoop/ozone/om/request/OMClientRequest.java | 39 +-
.../om/request/key/OMAllocateBlockRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 3 -
.../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 42 +-
.../multipart/S3MultipartUploadAbortRequest.java | 42 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 9 +-
.../S3MultipartUploadCommitPartRequest.java | 74 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 8 +-
.../S3MultipartUploadCompleteRequest.java | 78 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 +-
.../om/request/util/OMMultipartUploadUtils.java | 102 ++
.../AbstractS3MultipartAbortResponse.java | 53 +-
.../S3ExpiredMultipartUploadsAbortResponse.java | 3 +-
.../multipart/S3MultipartUploadAbortResponse.java | 30 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 14 +-
.../S3MultipartUploadCommitPartResponse.java | 23 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 8 +-
.../S3MultipartUploadCompleteResponse.java | 15 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 +-
.../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +-
.../ozone/om/snapshot/SnapshotDiffValueParser.java | 359 +++++
.../protocolPB/OzoneManagerRequestHandler.java | 13 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 33 +
.../TestOmRatisSnapshotProvider.java | 59 +
.../TestOMClientRequestUserInfoFallback.java | 106 --
.../om/request/key/TestOMKeyCommitRequest.java | 2 -
.../TestS3MultipartUploadCommitPartRequest.java | 355 +++++
.../s3/multipart/S3MultipartResponseTests.java | 10 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 4 +-
.../om/snapshot/TestSnapshotDiffValueParser.java | 249 ++++
.../protocolPB/TestOzoneManagerRequestHandler.java | 83 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 86 ++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 67 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 11 +
.../TestBasicRootedOzoneClientAdapterHeadOp.java | 176 +++
.../fs/ozone/TestRootedOzoneFileSystemHeadOp.java | 174 +++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 +
hadoop-ozone/pom.xml | 1 -
.../webapps/recon/ozone-recon-web/.eslintignore | 26 -
.../recon/ozone-recon-web/eslint.config.mjs | 109 ++
.../webapps/recon/ozone-recon-web/package.json | 15 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1111 +++++++--------
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +-
.../ozone/s3/commontypes/RequestParameters.java | 11 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 7 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 5 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 13 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 37 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 7 +-
.../signature/AccessDeniedResourceException.java | 12 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 +
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 4 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 17 +
.../hadoop/ozone/s3/endpoint/TestRootList.java | 94 ++
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 12 +-
.../s3/signature/TestAWSSignatureProcessor.java | 37 +
.../signature/TestAuthorizationV4QueryParser.java | 16 +-
hadoop-ozone/tools/pom.xml | 12 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 294 +++-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 9 +-
pom.xml | 32 +-
342 files changed, 9512 insertions(+), 7428 deletions(-)
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 289b5db922a,a0f1c1bf362..8b0ea537d4e
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@@ -38,9 -38,9 +38,11 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND_COUNT_UPDATED;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NEW_NODE;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
+ import static org.apache.ozone.test.MetricsAsserts.getLongCounter;
+ import static org.apache.ozone.test.MetricsAsserts.getMetrics;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
index bfce2f29097,2db0df2db61..8235e18fd05
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java
@@@ -17,8 -17,7 +17,7 @@@
package org.apache.hadoop.hdds.scm.node;
- import static java.lang.Thread.sleep;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
+import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager;
import static org.apache.ozone.test.MetricsAsserts.assertGauge;
import static org.apache.ozone.test.MetricsAsserts.getLongCounter;
import static org.apache.ozone.test.MetricsAsserts.getMetrics;
@@@ -38,9 -39,10 +37,10 @@@ import org.apache.hadoop.hdds.scm.HddsT
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+ import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@@ -238,8 -244,10 +238,9 @@@ public class TestSCMNodeMetrics
assertGauge("TotalFilesystemAvailable", 150L,
getMetrics(SCMNodeMetrics.class.getSimpleName()));
nodeManager.processHeartbeat(registeredDatanode);
- sleep(4000);
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5000);
metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
- assertGauge("InServiceHealthyReadonlyNodes", 0, metricsSource);
assertGauge("InServiceHealthyNodes", 1, metricsSource);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 77420a88216,ab4da4badd9..ca911f895c7
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@@ -134,7 -143,7 +143,7 @@@ public final class OmKeyInfo extends Wi
return new DelegatedCodec<>(
Proto2Codec.get(KeyInfo.getDefaultInstance()),
OmKeyInfo::getFromProtobuf,
- k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize()),
- k -> k.getProtobuf(true, ClientVersion.CURRENT_VERSION, isOpenKey),
++ k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize(), isOpenKey),
OmKeyInfo.class);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
index 2b882c55b7b,cf37681a793..39a3497aa1d
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
@@@ -55,7 -60,7 +60,7 @@@ public class RepeatedOmKeyInfo implemen
return new DelegatedCodec<>(
Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()),
RepeatedOmKeyInfo::getFromProto,
- k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize()),
- k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey),
++ k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize(), isOpenKey),
RepeatedOmKeyInfo.class);
}
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 3c079cd884f,e6c13160711..e9d0795db1e
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@@ -33,18 -33,16 +33,19 @@@ RESULT_DIR="$ALL_RESULT_DIR" create_res
# This is the version of Ozone that should use the runner image to run the
# code that was built. Other versions will pull images from docker hub.
- # run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION"
-run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
++# run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
+ # run_test ha non-rolling-upgrade 2.1.1 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION"
- #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
- #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
+ # run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
+ # run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.4.0 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.3.0 "$OZONE_CURRENT_VERSION"
# run_test ha non-rolling-upgrade 1.2.1 "$OZONE_CURRENT_VERSION"
# run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION"
- # Rolling upgrade test, commented out for now
- run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
++# Rolling upgrade test
++run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
+
generate_report "upgrade" "$ALL_RESULT_DIR"
exit "$RESULT"
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
index ae7e091658c,eea584643e6..3f735828b4c
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java
@@@ -41,17 -56,26 +41,17 @@@ import org.slf4j.LoggerFactory
/**
* Helper methods for testing HDDS upgrade finalization in integration tests.
*/
- public final class TestHddsUpgradeUtils {
+ public final class HddsUpgradeTestUtils {
- private static final Logger LOG = LoggerFactory.getLogger(TestHddsUpgradeUtils.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HddsUpgradeTestUtils.class);
- private TestHddsUpgradeUtils() { }
- private static final ReplicationConfig RATIS_THREE =
- ReplicationConfig.fromProtoTypeAndFactor(HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.THREE);
-
+ private HddsUpgradeTestUtils() { }
- public static void waitForFinalizationFromClient(
- StorageContainerLocationProtocol scmClient, String clientID)
- throws Exception {
+ public static void waitForFinalizationFromClient(StorageContainerLocationProtocol scmClient) throws Exception {
LambdaTestUtils.await(60_000, 1_000, () -> {
- UpgradeFinalization.Status status = scmClient
- .queryUpgradeFinalizationProgress(clientID, true, true)
- .status();
- LOG.info("Waiting for upgrade finalization to complete from client." +
- " Current status is {}.", status);
- return status == FINALIZATION_DONE || status == ALREADY_FINALIZED;
+ HddsProtos.UpgradeStatus status = scmClient.queryUpgradeStatus();
+ LOG.info("Waiting for upgrade finalization to complete from client. Current status is:\n{}", status);
+ return status.getShouldFinalize();
});
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
index 76bab76d66b,8c7daace078..a33af09e11f
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
@@@ -154,12 -167,24 +154,12 @@@ public class TestDNDataDistributionFina
// Validate pre-finalization state
validatePreDataDistributionFeatureState();
- // Now trigger finalization
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
-
// Wait for finalization to complete
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Verify finalization completed
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
// Create more data and deletions to test post-finalization behavior
String keyName3 = "testKey3";
@@@ -190,10 -215,21 +190,10 @@@
out.write(data);
}
bucket.deleteKey(keyName);
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- // Wait for finalization
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
-
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
// Verify the system can handle scenarios where pendingDeleteBlockCount
// might be missing and needs recalculation
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
index 6707cd87f00,3fb1bc121af..81fb21ebcde
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
@@@ -26,9 -26,9 +26,9 @@@ import static org.apache.hadoop.hdds.Hd
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT;
import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.EMPTY_SUMMARY;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@@ -160,18 -191,20 +160,18 @@@ public class TestScmDataDistributionFin
@Test
@Flaky("HDDS-14050")
public void testFinalizationEmptyClusterDataDistribution() throws Exception {
- init(new OzoneConfiguration(), null, true);
+ init(new OzoneConfiguration());
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
@@@ -266,15 -299,26 +266,15 @@@
flushDBTransactionBuffer(activeSCM);
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
index eff92e27589,c0e64ac4cac..802562f886a
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
@@@ -17,13 -17,21 +17,13 @@@
package org.apache.hadoop.hdds.upgrade;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmToFinalize;
- import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmToFinalize;
++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
+import org.apache.hadoop.hdds.HDDSVersion;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@@ -90,54 -126,118 +90,54 @@@ public class TestScmHAFinalization
}
}
- /**
- * Argument supplier for parameterized tests.
- */
- public static Stream<Arguments> injectionPointsToTest() {
- // Do not test from BEFORE_PRE_FINALIZE_UPGRADE injection point.
- // Finalization will not have started so there will be no persisted state
- // to resume from.
- return Stream.of(
- Arguments.of(UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE),
- Arguments.of(UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION),
- Arguments.of(UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE)
- );
- }
-
- @ParameterizedTest
- @MethodSource(METHOD_SOURCE)
- public void testFinalizationWithLeaderChange(
- UpgradeTestInjectionPoints haltingPoint) throws Exception {
-
- CountDownLatch pauseLatch = new CountDownLatch(1);
- CountDownLatch unpauseLatch = new CountDownLatch(1);
- init(new OzoneConfiguration(),
- UpgradeTestUtils.newPausingFinalizationExecutor(haltingPoint,
- pauseLatch, unpauseLatch, LOG), 0);
- pauseLatch.await();
-
- // Stop the leader, forcing a leader change in the middle of finalization.
- // This will cause the initial client call for finalization
- // to be interrupted.
- StorageContainerManager oldLeaderScm = cluster.getActiveSCM();
- LOG.info("Stopping current SCM leader {} to initiate a leader change.",
- oldLeaderScm.getSCMNodeId());
- cluster.shutdownStorageContainerManager(oldLeaderScm);
-
- // Wait for the remaining two SCMs to elect a new leader.
- cluster.waitForClusterToBeReady();
-
- // While finalization is paused, check its state on the remaining SCMs.
- checkMidFinalizationConditions(haltingPoint,
- cluster.getStorageContainerManagersList());
-
- // Restart actually creates a new SCM.
- // Since this SCM will be a follower, the implementation of its upgrade
- // finalization executor does not matter for this test.
- cluster.restartStorageContainerManager(oldLeaderScm, true);
-
- // Make sure the original SCM leader is not the leader anymore.
- StorageContainerManager newLeaderScm = cluster.getActiveSCM();
- assertNotEquals(newLeaderScm.getSCMNodeId(),
- oldLeaderScm.getSCMNodeId());
-
- // Resume finalization from the new leader.
- unpauseLatch.countDown();
-
- // Client should complete exceptionally since the original SCM it
- // requested to was restarted.
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- // Make sure old leader has caught up and all SCMs have finalized.
- waitForScmsToFinalize(cluster.getStorageContainerManagersList());
-
- HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
- HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
- }
-
- @ParameterizedTest
- @MethodSource(METHOD_SOURCE)
- @Flaky("HDDS-8714")
- public void testFinalizationWithRestart(
- UpgradeTestInjectionPoints haltingPoint) throws Exception {
- CountDownLatch terminateLatch = new CountDownLatch(1);
- init(new OzoneConfiguration(),
- UpgradeTestUtils.newTerminatingFinalizationExecutor(haltingPoint,
- terminateLatch, LOG),
- 0);
- terminateLatch.await();
+ @Test
+ public void testFinalizedDatanodesShutDownWithPrefinalizedScm() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(SCMStorageConfig.TESTING_INIT_APPARENT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.serialize());
+ conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT, "5s");
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT, "1s");
- // Once upgrade finalization is stopped at the halting point, restart all
- // SCMs.
- LOG.info("Restarting all SCMs during upgrade finalization.");
- // Restarting an SCM from mini ozone actually replaces the SCM with a new
- // instance. We will use the normal upgrade finalization executor for
- // these new instances, since the last one aborted at the halting point.
- cluster.getSCMConfigurator()
- .setUpgradeFinalizationExecutor(
- new DefaultUpgradeFinalizationExecutor<>());
- List<StorageContainerManager> originalSCMs =
- cluster.getStorageContainerManagers();
+ MiniOzoneHAClusterImpl.Builder clusterBuilder = MiniOzoneCluster.newHABuilder(conf);
+ clusterBuilder.setNumOfStorageContainerManagers(NUM_SCMS)
+ .setNumOfActiveSCMs(NUM_SCMS)
+ .setSCMServiceId("scmservice")
+ .setNumOfOzoneManagers(1)
+ .setNumDatanodes(NUM_DATANODES)
+ .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
+ .setApparentVersion(HDDSVersion.SOFTWARE_VERSION.serialize())
+ .build());
- for (StorageContainerManager scm: originalSCMs) {
- cluster.restartStorageContainerManager(scm, false);
+ // Prevent terminateDatanode() from calling System.exit(1) and killing the test JVM.
+ ExitUtil.disableSystemExit();
+ LogCapturer logCapture = LogCapturer.captureLogs(RegisterEndpointTask.class);
+ // This starts the mini ozone cluster.
+ cluster = clusterBuilder.build();
+
+ // isStopped cannot be set to true unless a datanode was started first.
+ // Each datanode should be rejected since its apparent version exceeds the pre-finalized SCM's.
+ GenericTestUtils.waitFor(
+ () -> cluster.getHddsDatanodes().stream().allMatch(HddsDatanodeService::isStopped),
+ 500, 30_000);
+
+ assertThat(logCapture.getOutput()).contains("SCM rejected this datanode's registration");
+ for (StorageContainerManager scm : cluster.getStorageContainerManagersList()) {
+ assertThat(scm.getScmNodeManager().getAllNodes()).isEmpty();
}
+ }
- checkMidFinalizationConditions(haltingPoint,
- cluster.getStorageContainerManagersList());
-
- // After all SCMs were restarted, finalization should resume
- // automatically once a leader is elected.
- cluster.waitForClusterToBeReady();
-
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- // Once the leader tells the client finalization is complete, wait for all
- // followers to catch up so we can check their state.
+ @Test
+ public void testFinalization() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ init(conf, 0);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+ // Ensure all SCMs finalize, indicating the message has been propagated across them all
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
}
@Test
@@@ -167,15 -267,15 +167,15 @@@
}
// Wait for finalization from the client perspective.
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Wait for two running SCMs to finish finalization.
waitForScmsToFinalize(activeScms);
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- activeScms, 0, NUM_DATANODES);
+ activeScms, 0);
- TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
// Move SCM log index farther ahead to make sure a snapshot install
// happens on the restarted SCM.
@@@ -188,16 -288,90 +188,16 @@@
}
cluster.startInactiveSCM(inactiveScm.getSCMNodeId());
- waitForScmToFinalize(inactiveScm);
+ LOG.info("Waiting for restarted SCM to finalize");
+ // When the leader sends a snapshot to the follower, it should have flushed all entries to the DB, including the
+ // apparent versin. This means the follower should see it in the DB it receives immediately to trigger finalization.
+ waitForScmToFinalize(inactiveScm, true);
- TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
+ HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- inactiveScm, 0, NUM_DATANODES);
+ inactiveScm, 0);
// Use log to verify a snapshot was installed.
- assertThat(logCapture.getOutput()).contains("New SCM snapshot " +
- "received with metadata layout version");
- }
-
- private void waitForScmsToFinalize(Collection<StorageContainerManager> scms)
- throws Exception {
- for (StorageContainerManager scm: scms) {
- waitForScmToFinalize(scm);
- }
- }
-
- private void waitForScmToFinalize(StorageContainerManager scm)
- throws Exception {
- GenericTestUtils.waitFor(() -> !scm.isInSafeMode(), 500, 5000);
- GenericTestUtils.waitFor(() -> {
- FinalizationCheckpoint checkpoint =
- scm.getScmContext().getFinalizationCheckpoint();
- LOG.info("Waiting for SCM {} (leader? {}) to finalize. Current " +
- "finalization checkpoint is {}",
- scm.getSCMNodeId(), scm.checkLeader(), checkpoint);
- return checkpoint.hasCrossed(
- FinalizationCheckpoint.FINALIZATION_COMPLETE);
- }, 2_000, 60_000);
- }
-
- private void checkMidFinalizationConditions(
- UpgradeTestInjectionPoints haltingPoint,
- List<StorageContainerManager> scms) {
-
- // Ratis only makes sure that the Leader has processed the finalization,
- // the followers might have this in the Raft Log and not yet processed it.
- switch (haltingPoint) {
- case BEFORE_PRE_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_REQUIRED stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_REQUIRED));
- // Pipeline creation should not be frozen at this point, even on leader.
- assertTrue(scms.stream().noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_PRE_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_STARTED stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED));
- // Pipeline creation should be frozen on nodes where the finalization checkpoint is FINALIZATION_STARTED,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED)
- .allMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_COMPLETE_FINALIZATION:
- // At least one node (leader) should be in the MLV_EQUALS_SLV stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV));
- // Pipeline creation should not be frozen on nodes where the finalization checkpoint is MLV_EQUALS_SLV,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV)
- .noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- case AFTER_POST_FINALIZE_UPGRADE:
- // At least one node (leader) should be in the FINALIZATION_COMPLETE stage.
- assertTrue(scms.stream().anyMatch(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE));
- // Pipeline creation should not be frozen on nodes where the finalization checkpoint is FINALIZATION_COMPLETE,
- // this should include the leader SCM.
- assertTrue(scms.stream()
- .filter(scm ->
- scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE)
- .noneMatch(scm ->
- scm.getPipelineManager().isPipelineCreationFrozen()));
- break;
- default:
- fail("Unknown halting point in test: " + haltingPoint);
- }
+ assertThat(logCapture.getOutput()).contains("New snapshot received with higher apparent version " +
+ HDDSVersion.SOFTWARE_VERSION + ". Attempting to finalize to that version.");
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
index 482662c9df8,3c1d3ed6faf..0c3bc3b37bc
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
@@@ -41,9 -45,11 +41,9 @@@ import org.apache.hadoop.hdds.conf.Ozon
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMPerformanceMetrics;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
- import org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils;
-import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
+ import org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.UniformDatanodesFactory;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@@ -136,8 -151,18 +136,8 @@@ public class TestBlockDeletionService
GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
// UPGRADE SCM (if specified)
- // Step 5: wait for finalizing upgrade
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(() -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- fail("finalization client failed", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
- assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(),
- cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+ scmClient.finalizeUpgrade();
- TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient);
++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// POST-UPGRADE
//Step 6: Repeat the same steps in pre-upgrade
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
index f6e0baec5e5,44131d81c88..aea70b1d253
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
@@@ -51,8 -51,11 +51,9 @@@ import static org.apache.hadoop.ozone.s
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
import static org.apache.ozone.test.LambdaTestUtils.await;
+ import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
index 5bfbe349e83,884c5fa310b..f913b7a79ad
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java
@@@ -111,12 -111,12 +111,12 @@@ public abstract class OMClientRequest i
*/
public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
- LayoutVersion layoutVersion = LayoutVersion.newBuilder()
- .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion())
+ LayoutVersion apparentVersion = LayoutVersion.newBuilder()
+ .setVersion(ozoneManager.getVersionManager().getApparentVersion().serialize())
.build();
omRequest = getOmRequest().toBuilder()
- .setUserInfo(getUserIfNotExists(ozoneManager))
- .setLayoutVersion(layoutVersion).build();
+ .setLayoutVersion(apparentVersion)
- .setUserInfo(getUserInfo()).build();
++ .setUserInfo(getUserIfNotExists(ozoneManager)).build();
return omRequest;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org