You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2024/06/21 10:51:28 UTC
(ozone) 01/01: Merge remote-tracking branch 'origin/master' into HDDS-10656-atomic-key-overwrite
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-10656-atomic-key-overwrite
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1fe90111a3fbdbdf0101b200f14dc7227c982fcc
Merge: 99be3179a4 9f1f7ed238
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Fri Jun 21 10:30:37 2024 +0200
Merge remote-tracking branch 'origin/master' into HDDS-10656-atomic-key-overwrite
.github/workflows/build-ratis.yml | 137 +++
.github/workflows/ci-with-ratis.yml | 52 +
.github/workflows/ci.yml | 66 +-
.github/workflows/intermittent-test-check.yml | 62 +-
.github/workflows/populate-cache.yml | 24 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.gitignore | 1 +
.mvn/{gradle-enterprise.xml => develocity.xml} | 15 +-
.mvn/extensions.xml | 6 +-
dev-support/ci/download-nodejs.sh | 63 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 6 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 68 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 59 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 92 +-
.../hadoop/hdds/scm/client/ClientTrustManager.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 31 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 25 +-
.../ozone/client/io/BlockInputStreamFactory.java | 3 +-
.../client/io/BlockInputStreamFactoryImpl.java | 3 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 7 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
.../hdds/scm/storage/TestBlockInputStream.java | 2 +-
.../storage/TestBlockOutputStreamCorrectness.java | 10 +-
.../client/io/TestBlockInputStreamFactoryImpl.java | 11 +-
hadoop-hdds/common/pom.xml | 12 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 -
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 57 +-
.../hadoop/hdds/client/DecommissionUtils.java | 153 +++
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 124 ++-
.../org/apache/hadoop/hdds/client/QuotaList.java | 69 --
.../hadoop/hdds/conf/OzoneConfiguration.java | 9 +-
.../hadoop/hdds/freon/FakeClusterTopology.java | 38 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 135 ++-
.../hdds/ratis/ContainerCommandRequestMessage.java | 4 +
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 7 +
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 22 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +-
.../java/org/apache/hadoop/hdds/scm/ScmInfo.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientReply.java | 3 +-
.../scm/container/common/helpers/ExcludeList.java | 19 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 6 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +-
.../apache/hadoop/hdds/scm/net/NetConstants.java | 3 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 55 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 29 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 53 +-
.../hadoop/hdds/security/SecurityConfig.java | 51 +-
.../x509/certificate/utils/CertificateCodec.java | 107 +-
.../certificate/utils/SelfSignedCertificate.java | 24 +-
.../hadoop/hdds/security/x509/crl/CRLStatus.java | 87 --
.../hdds/security/x509/crl/package-info.java | 24 -
.../org/apache/hadoop/hdds/server/JsonUtils.java | 34 +-
.../apache/hadoop/hdds/utils/BatchOperation.java | 88 --
.../org/apache/hadoop/ozone/ClientVersion.java | 4 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +-
.../apache/hadoop/ozone/OzoneManagerVersion.java | 4 +-
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 5 +-
.../apache/hadoop/ozone/common/ChecksumData.java | 5 +-
.../ozone/common/statemachine/StateMachine.java | 10 +-
.../ozone/container/common/helpers/ChunkInfo.java | 13 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 26 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 14 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 5 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 38 +-
.../org/apache/hadoop/util/PerformanceMetrics.java | 23 +-
.../hadoop/util/PerformanceMetricsInitializer.java | 24 +-
.../apache/hadoop/util/StringWithByteString.java | 54 +
.../common/src/main/resources/ozone-default.xml | 15 +-
.../org/apache/hadoop/hdds/JsonTestUtils.java} | 69 +-
.../apache/hadoop/hdds/client/TestOzoneQuota.java} | 24 +-
.../ratis/TestContainerCommandRequestMessage.java | 3 +
.../hdds/security/x509/CertificateTestUtils.java | 1 +
.../ozone/container/ContainerTestHelper.java | 22 +-
.../hdds/datanode/metadata/CRLDBDefinition.java | 100 --
.../hdds/datanode/metadata/DatanodeCRLStore.java | 84 --
.../datanode/metadata/DatanodeCRLStoreImpl.java | 128 ---
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../container/common/helpers/ContainerMetrics.java | 61 +-
.../container/common/helpers/DatanodeIdYaml.java | 6 +-
.../container/common/impl/HddsDispatcher.java | 3 +-
.../common/report/CRLStatusReportPublisher.java | 82 --
.../common/report/ReportPublisherFactory.java | 2 -
.../common/statemachine/DatanodeStateMachine.java | 12 +-
.../common/statemachine/StateContext.java | 13 -
.../ClosePipelineCommandHandler.java | 32 +-
.../common/transport/server/ratis/CSMMetrics.java | 16 +-
.../server/ratis/ContainerStateMachine.java | 45 +-
.../ozone/container/common/volume/VolumeUsage.java | 39 +-
.../KeyValueContainerMetadataInspector.java | 178 ++--
.../ozone/container/keyvalue/KeyValueHandler.java | 23 +-
.../container/keyvalue/helpers/BlockUtils.java | 23 +-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 20 +-
.../keyvalue/impl/KeyValueStreamDataChannel.java | 7 +-
.../keyvalue/impl/StreamDataChannelBase.java | 3 +
.../keyvalue/interfaces/BlockManager.java | 4 +-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../metadata/TestDatanodeCRLStoreImpl.java | 117 --
.../hdds/datanode/metadata/package-info.java | 22 -
.../hadoop/ozone/TestHddsDatanodeService.java | 24 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 59 +-
.../container/common/helpers/TestBlockData.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../common/report/TestReportPublisher.java | 74 --
.../common/report/TestReportPublisherFactory.java | 12 -
.../common/statemachine/TestStateContext.java | 11 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 4 +
.../common/volume/TestReservedVolumeSpace.java | 60 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 5 +
.../TestKeyValueContainerMetadataInspector.java | 94 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 59 ++
.../impl/TestKeyValueStreamDataChannel.java | 4 +-
hadoop-hdds/crypto-api/pom.xml | 37 +
hadoop-hdds/crypto-default/pom.xml | 37 +
hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +-
.../content/concept/StorageContainerManager.md | 2 +-
hadoop-hdds/docs/content/start/OnPrem.md | 2 +-
hadoop-hdds/docs/content/start/OnPrem.zh.md | 6 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 2 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 8 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 10 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 31 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 61 +-
.../DeletedBlocksTransactionInfoWrapper.java | 9 +-
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 46 -
.../scm/update/client/CRLClientUpdateHandler.java | 202 ----
.../hadoop/hdds/scm/update/client/CRLStore.java | 34 -
.../hdds/scm/update/client/ClientCRLStore.java | 98 --
.../scm/update/client/ClientUpdateHandler.java | 29 -
.../client/SCMUpdateClientConfiguration.java | 53 -
.../update/client/SCMUpdateServiceGrpcClient.java | 219 ----
.../scm/update/client/UpdateServiceConfig.java | 43 -
.../hdds/scm/update/client/package-info.java | 22 -
.../hdds/scm/update/server/CRLClientInfo.java | 56 -
.../scm/update/server/SCMUpdateClientInfo.java | 67 --
.../hdds/scm/update/server/package-info.java | 22 -
.../security/token/CompositeTokenVerifier.java | 4 +-
.../hdds/security/token/NoopTokenVerifier.java | 4 +-
.../security/token/ShortLivedTokenVerifier.java | 8 +-
.../hadoop/hdds/security/token/TokenVerifier.java | 9 +-
.../x509/certificate/authority/BaseApprover.java | 18 +-
.../x509/certificate/authority/CRLApprover.java | 43 -
.../certificate/authority/CertificateApprover.java | 24 +-
.../certificate/authority/CertificateServer.java | 46 +-
.../certificate/authority/CertificateStore.java | 93 +-
.../certificate/authority/DefaultApprover.java | 26 +-
.../certificate/authority/DefaultCAServer.java | 164 +--
.../certificate/authority/DefaultCRLApprover.java | 58 -
.../client/DefaultCertificateClient.java | 2 +-
.../certificate/client/SCMCertificateClient.java | 19 +-
.../hadoop/hdds/security/x509/crl/CRLCodec.java | 220 ----
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 226 ----
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 4 +
.../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +-
.../hadoop/hdds/conf/TestHddsConfServlet.java | 5 +-
.../token/TestOzoneBlockTokenSecretManager.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 14 +-
.../x509/certificate/authority/MockApprover.java | 58 -
.../x509/certificate/authority/MockCAStore.java | 56 +-
.../certificate/authority/TestDefaultCAServer.java | 135 +--
.../certificate/authority/TestDefaultProfile.java | 33 +-
.../client/CertificateClientTestImpl.java | 64 +-
.../client/TestDefaultCertificateClient.java | 12 +-
.../client/TestDnCertificateClientInit.java | 7 +-
.../client/TestRootCaRotationPoller.java | 15 +-
.../x509/certificate/utils/TestCRLCodec.java | 288 -----
.../certificate/utils/TestCertificateCodec.java | 174 +--
.../certificate/utils/TestRootCertificate.java | 69 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 5 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 39 +
.../src/main/proto/DatanodeClientProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 3 +
.../src/main/proto/ScmServerSecurityProtocol.proto | 20 +
.../utils/db/managed/ManagedSstFileReader.java | 21 +-
hadoop-hdds/pom.xml | 2 +
.../src/main/patches/rocks-native.patch | 5 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 37 -
.../ozone/rocksdb/util/SstFileSetReader.java | 33 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 104 +-
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 9 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 107 +-
.../ContainerBalancerSelectionCriteria.java | 18 +-
.../container/balancer/ContainerBalancerTask.java | 104 +-
.../container/balancer/ContainerMoveSelection.java | 22 +
.../algorithms/SCMContainerPlacementCapacity.java | 6 +
.../algorithms/SCMContainerPlacementRandom.java | 6 +
.../placement/metrics/SCMPerformanceMetrics.java | 94 ++
.../container/replication/ReplicationManager.java | 6 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 -
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 19 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 9 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 -
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 45 -
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 77 --
.../hdds/scm/node/NodeDecommissionManager.java | 28 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 7 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 13 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 71 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 31 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 60 +-
.../hdds/scm/security/CRLStatusReportHandler.java | 87 --
.../hdds/scm/security/RootCARotationManager.java | 36 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 8 +
.../hadoop/hdds/scm/server/SCMCertStore.java | 205 +---
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 14 -
.../hdds/scm/server/SCMSecurityProtocolServer.java | 71 +-
.../hdds/scm/server/StorageContainerManager.java | 54 +-
.../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 -
.../scm/update/server/SCMCRLUpdateHandler.java | 142 ---
.../scm/update/server/SCMUpdateClientManager.java | 149 ---
.../hdds/scm/update/server/SCMUpdateHandler.java | 53 -
.../update/server/SCMUpdateServiceGrpcServer.java | 100 --
.../scm/update/server/SCMUpdateServiceImpl.java | 119 ---
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 16 -
.../hdds/scm/container/balancer/MockedSCM.java | 35 +-
.../TestContainerBalancerDatanodeNodeLimit.java | 165 ++-
.../balancer/TestContainerBalancerTask.java | 167 +--
.../scm/container/balancer/TestableCluster.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 63 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 8 +
.../scm/metadata/TestX509CertificateCodec.java | 8 +
.../scm/security/TestCRLStatusReportHandler.java | 137 ---
.../scm/security/TestRootCARotationManager.java | 7 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 198 +---
.../hdds/scm/update/server/MockCRLStore.java | 140 ---
.../server/TestSCMUpdateServiceGrpcServer.java | 302 ------
.../ozone/container/common/TestEndPoint.java | 45 +-
.../placement/TestContainerPlacement.java | 6 +-
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 15 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 50 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 50 +-
.../scm/cli/pipeline/FilterPipelineOptions.java | 85 ++
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 77 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 92 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 96 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 25 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 8 +-
.../client/checksum/ECBlockChecksumComputer.java | 6 +-
.../client/checksum/ECFileChecksumHelper.java | 7 +-
.../checksum/ReplicatedFileChecksumHelper.java | 8 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 4 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 8 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 74 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 177 +++-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 37 +-
.../ozone/om/exceptions/OMNotLeaderException.java | 7 +
.../apache/hadoop/ozone/om/helpers/ErrorInfo.java | 33 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 20 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +-
.../apache/hadoop/ozone/om/helpers/WithTags.java | 20 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 26 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 4 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 50 +-
.../ozone/snapshot/CancelSnapshotDiffResponse.java | 2 +-
.../ozone/snapshot/ListSnapshotResponse.java | 45 +-
.../ozone/snapshot/SnapshotDiffReportOzone.java | 24 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 4 +-
hadoop-ozone/dev-support/checks/blockade.sh | 2 +-
hadoop-ozone/dev-support/checks/coverage.sh | 4 +-
hadoop-ozone/dev-support/checks/junit.sh | 6 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +-
hadoop-ozone/dev-support/checks/native.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 1 -
.../dist/dev-support/bin/dist-layout-stitching | 3 +
.../dist/src/main/compose/common/s3a-test.sh | 3 +-
.../ps.sh => compose_v2_compatibility.sh} | 9 +-
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 2 +
.../dist/src/main/compose/ozonescripts/README.md | 2 +-
.../dist/src/main/compose/ozonescripts/ps.sh | 5 +-
.../dist/src/main/compose/ozonescripts/start.sh | 3 +
.../dist/src/main/compose/ozonescripts/stop.sh | 3 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 29 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +-
.../dist/src/main/keytabs/update-keytabs.sh | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 3 +
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../dist/src/main/smoketest/admincli/cert.robot | 4 +-
.../dist/src/main/smoketest/env-compose.robot | 32 -
.../main/smoketest/httpfs/operations_tests.robot | 10 -
.../src/main/smoketest/s3/MultipartUpload.robot | 25 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 21 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 32 +-
.../src/main/smoketest/security/admin-cert.robot | 5 +-
.../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +
.../dist/src/shell/ozone/ozone-functions.sh | 5 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 12 +-
.../dev-support/findbugsExcludeFile.xml | 14 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 37 +
.../ozone/AbstractOzoneFileSystemTestWithFSO.java | 2 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 6 +
.../hadoop/hdds/scm/TestContainerSmallFile.java | 2 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 163 ---
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 -
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 22 +-
.../TestContainerStateManagerIntegration.java | 5 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 14 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 10 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 -
.../hadoop/ozone/TestContainerOperations.java | 56 +
.../TestContainerStateMachineIdempotency.java | 116 --
.../apache/hadoop/ozone/TestDelegationToken.java | 2 -
.../hadoop/ozone/TestMultipartObjectGet.java | 10 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 157 ++-
...lientAbstract.java => OzoneRpcClientTests.java} | 514 +++++++--
.../ozone/client/rpc/TestBlockOutputStream.java | 75 +-
...ayCommitInRatis.java => TestCommitInRatis.java} | 30 +-
.../client/rpc/TestFailureHandlingByClient.java | 19 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 8 -
.../ozone/client/rpc/TestOzoneRpcClient.java | 16 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 348 +-----
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 125 +--
.../ozone/client/rpc/TestWatchForCommit.java | 198 ++--
.../ozone/container/TestContainerReplication.java | 209 ++++
.../ozoneimpl/TestOzoneContainerWithTLS.java | 7 +
.../org/apache/hadoop/ozone/debug/TestLDBCli.java | 24 +
.../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 35 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 9 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 28 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +-
.../ozone/recon/TestReconContainerEndpoint.java | 222 ++++
.../hadoop/ozone/recon/TestReconScmHASnapshot.java | 2 +-
...napshot.java => TestReconScmNonHASnapshot.java} | 11 +-
.../ozone/recon/TestReconWithOzoneManager.java | 10 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 20 +-
.../shell/TestOzoneContainerUpgradeShell.java | 2 +
.../hadoop/ozone/shell/TestOzoneRepairShell.java | 133 +++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 26 +-
.../hadoop/ozone/shell/TestReconfigShell.java | 3 +
.../src/main/proto/OmClientProtocol.proto | 17 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 54 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 40 +-
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 12 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +
.../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 9 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 17 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 12 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 3 +
.../S3InitiateMultipartUploadRequest.java | 3 +
.../S3InitiateMultipartUploadRequestWithFSO.java | 3 +
.../multipart/S3MultipartUploadAbortRequest.java | 2 +
.../S3MultipartUploadCommitPartRequest.java | 1 +
.../S3MultipartUploadCompleteRequest.java | 7 +-
.../protocolPB/OzoneManagerRequestHandler.java | 18 +-
.../OzoneDelegationTokenSecretManager.java | 2 +
.../apache/hadoop/ozone/om/TestChunkStreams.java | 5 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 61 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 20 +
.../ozone/om/request/OMRequestTestUtils.java | 7 +-
.../om/request/key/TestOMKeyCreateRequest.java | 62 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 7 +
.../TestS3InitiateMultipartUploadRequest.java | 9 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 9 +-
.../s3/multipart/TestS3MultipartRequest.java | 43 +-
.../TestS3MultipartUploadCompleteRequest.java | 20 +-
.../security/TestOmCertificateClientInit.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 12 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 24 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 6 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 7 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 6 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 9 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 1 +
.../apache/hadoop/ozone/recon/ReconContext.java | 149 +++
.../hadoop/ozone/recon/ReconControllerModule.java | 1 +
.../hadoop/ozone/recon/ReconResponseUtils.java | 84 ++
.../org/apache/hadoop/ozone/recon/ReconServer.java | 16 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 1 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 221 +++-
.../ozone/recon/api/ClusterStateEndpoint.java | 13 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 51 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 25 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 137 ++-
.../ozone/recon/api/OMDBInsightEndpoint.java | 655 +++++++++++-
.../recon/api/handlers/DirectoryEntityHandler.java | 1 -
.../ozone/recon/api/handlers/EntityHandler.java | 2 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 1 -
.../recon/api/types/ClusterStateResponse.java | 29 +
.../ozone/recon/api/types/DatanodeMetrics.java | 81 ++
.../api/types/DecommissionStatusInfoResponse.java | 73 ++
.../ozone/recon/api/types/KeyEntityInfo.java | 42 +-
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 11 +
.../hadoop/ozone/recon/api/types/KeysResponse.java | 9 +-
.../ozone/recon/api/types/ListKeysResponse.java | 114 ++
.../hadoop/ozone/recon/api/types/NSSummary.java | 15 +-
.../hadoop/ozone/recon/api/types/ParamInfo.java | 133 +++
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 17 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 14 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 2 +-
.../ozone/recon/scm/ReconContainerManager.java | 8 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 66 +-
.../ozone/recon/scm/ReconPipelineManager.java | 79 +-
.../recon/scm/ReconPipelineReportHandler.java | 7 +-
.../scm/ReconStorageContainerManagerFacade.java | 29 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 3 +
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 12 +-
.../impl/StorageContainerServiceProviderImpl.java | 110 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/ContainerSizeCountTask.java | 6 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 6 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 2 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 2 +
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 4 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 16 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 6 +-
.../ozone/recon/tasks/OmTableInsightTask.java | 4 +-
.../webapps/recon/ozone-recon-web/api/db.json | 296 +++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 31 +-
.../src/components/overviewCard/overviewCard.tsx | 5 +
.../src/components/rightDrawer/rightDrawer.tsx | 4 +-
.../src/utils/axiosRequestHelper.tsx | 15 +
.../src/views/datanodes/datanodes.less | 4 +
.../src/views/datanodes/datanodes.tsx | 84 +-
.../src/views/diskUsage/diskUsage.tsx | 41 +-
.../ozone-recon-web/src/views/heatMap/heatmap.tsx | 65 +-
.../src/views/overview/overview.tsx | 23 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 57 +-
...teCounts.java => TestClusterStateEndpoint.java} | 28 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 164 ++-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 166 ++-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 48 +-
.../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 76 +-
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 1116 +++++++++++++++++++-
.../ozone/recon/heatmap/TestHeatMapInfo.java | 440 ++++----
.../ozone/recon/scm/TestReconNodeManager.java | 50 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 36 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 54 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 61 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 6 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 84 ++
.../endpoint/MultiDeleteRequestUnmarshaller.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 93 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 5 +-
.../endpoint/PutBucketAclRequestUnmarshaller.java | 6 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 14 +
.../hadoop/ozone/client/ClientProtocolStub.java | 38 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 46 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 18 -
...CompleteMultipartUploadRequestUnmarshaller.java | 44 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 75 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 229 +++-
.../ozone/s3/endpoint/TestPermissionCheck.java | 13 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 4 -
.../ozone/admin/nssummary/NSSummaryAdmin.java | 6 +
.../ozone/admin/om/DecommissionOMSubcommand.java | 4 +-
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 15 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 26 +-
.../apache/hadoop/ozone/debug/RocksDBUtils.java | 31 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 15 +-
.../hadoop/ozone/freon/DNRPCLoadGenerator.java | 31 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 4 +
.../freon/OzoneClientKeyReadWriteListOps.java | 5 +
.../GenerateOzoneRequiredConfigurations.java | 3 +-
.../apache/hadoop/ozone/repair/OzoneRepair.java | 30 +
.../hadoop/ozone/repair/TransactionInfoRepair.java | 135 +++
.../hadoop/ozone/repair/om/SnapshotRepair.java | 34 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 37 +-
.../ozone/shell/snapshot/ListSnapshotHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 78 +-
.../hadoop/ozone/repair/TestOzoneRepair.java | 88 ++
.../ozone/repair/TestTransactionInfoRepair.java | 146 +++
.../shell/TestOzoneAddressClientCreation.java | 30 +
.../org.mockito.plugins.MockMaker} | 3 +-
pom.xml | 121 ++-
479 files changed, 11745 insertions(+), 9772 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index f5031a1265,c55945d537..c53aca2f15
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@@ -40,8 -40,8 +40,10 @@@ public enum OzoneManagerVersion impleme
LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight"
+ " listKeys API."),
- ATOMIC_REWRITE_KEY(5, "OzoneManager version that supports rewriting key as atomic operation"),
+ OBJECT_TAG(5, "OzoneManager version that supports object tags"),
+
++ ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"),
+
FUTURE_VERSION(-1, "Used internally in the client when the server side is "
+ " newer and an unknown server version has arrived to the client.");
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 012f029f51,575701be80..216b51b8e8
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@@ -466,32 -467,29 +467,51 @@@ public class OzoneBucket extends WithMe
ReplicationConfig replicationConfig,
Map<String, String> keyMetadata)
throws IOException {
+ return this.createKey(key, size, replicationConfig, keyMetadata, Collections.emptyMap());
+ }
+
+ /**
+ * Creates a new key in the bucket.
+ *
+ * @param key Name of the key to be created.
+ * @param size Size of the data the key will point to.
+ * @param replicationConfig Replication configuration.
+ * @param keyMetadata Custom key metadata.
+ * @param tags Tags used for S3 object tags
+ * @return OzoneOutputStream to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneOutputStream createKey(String key, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata,
+ Map<String, String> tags)
+ throws IOException {
return proxy
- .createKey(volumeName, name, key, size, replicationConfig, keyMetadata);
+ .createKey(volumeName, name, key, size, replicationConfig, keyMetadata, tags);
}
+ /**
+ * This API allows to atomically update an existing key. The key read before invoking this API
+ * should remain unchanged for this key to be written. This is controlled by the generation
+ * field in the existing Key param. If the key is replaced or updated the generation will change. If the
+ * generation has changed since the existing Key was read, either the initial key create will fail,
+ * or the key will fail to commit after the data has been written as the checks are carried out
+ * both at key open and commit time.
+ *
+ * @param keyName Existing key to rewrite. This must exist in the bucket.
+ * @param size The size of the new key
+ * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create
+ * and commit time.
+ * @param replicationConfig The replication configuration for the key to be rewritten.
+ * @param metadata custom key value metadata
+ * @return OzoneOutputStream to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyGeneration,
+ ReplicationConfig replicationConfig, Map<String, String> metadata) throws IOException {
+ return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata);
+ }
+
/**
* Creates a new key in the bucket, with default replication type RATIS and
* with replication factor THREE.
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
index cd2978fce1,168e15d9bd..fe547794e9
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
@@@ -59,30 -53,12 +59,30 @@@ public class OzoneKeyDetails extends Oz
Map<String, String> metadata,
FileEncryptionInfo feInfo,
CheckedSupplier<OzoneInputStream, IOException> contentSupplier,
- boolean isFile, String owner, Long generation) {
- boolean isFile, String owner, Map<String, String> tags) {
++ boolean isFile, String owner, Map<String, String> tags, Long generation) {
super(volumeName, bucketName, keyName, size, creationTime,
- modificationTime, replicationConfig, metadata, isFile, owner);
+ modificationTime, replicationConfig, metadata, isFile, owner, tags);
this.ozoneKeyLocations = ozoneKeyLocations;
this.feInfo = feInfo;
this.contentSupplier = contentSupplier;
+ this.generation = generation;
+ }
+
+ /**
+ * Constructs OzoneKeyDetails from OmKeyInfo.
+ */
+ @SuppressWarnings("parameternumber")
+ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
+ long size, long creationTime, long modificationTime,
+ List<OzoneKeyLocation> ozoneKeyLocations,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata,
+ FileEncryptionInfo feInfo,
+ CheckedSupplier<OzoneInputStream, IOException> contentSupplier,
- boolean isFile, String owner) {
++ boolean isFile, String owner, Map<String, String> tags) {
+ this(volumeName, bucketName, keyName, size, creationTime,
+ modificationTime, ozoneKeyLocations, replicationConfig, metadata, feInfo, contentSupplier,
- isFile, owner, null);
++ isFile, owner, tags, null);
}
/**
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 0725b4f253,74344344f8..55985a18d3
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@@ -353,29 -355,22 +355,45 @@@ public interface ClientProtocol
Map<String, String> metadata)
throws IOException;
+ /**
+ * This API allows to atomically update an existing key. The key read before invoking this API
+ * should remain unchanged for this key to be written. This is controlled by the generation
+ * field in the existing Key param. If the key is replaced or updated the generation will change. If the
+ * generation has changed since the existing Key was read, either the initial key create will fail,
+ * or the key will fail to commit after the data has been written as the checks are carried out
+ * both at key open and commit time.
+ *
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Existing key to rewrite. This must exist in the bucket.
+ * @param size The size of the new key
+ * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create
+ * and commit time.
+ * @param replicationConfig The replication configuration for the key to be rewritten.
+ * @param metadata custom key value metadata
+ * @return {@link OzoneOutputStream}
+ * @throws IOException
+ */
+ OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName,
+ long size, long existingKeyGeneration, ReplicationConfig replicationConfig,
+ Map<String, String> metadata) throws IOException;
+
+ /**
+ * Writes a key in an existing bucket.
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param metadata Custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return {@link OzoneOutputStream}
+ *
+ */
+ OzoneOutputStream createKey(String volumeName, String bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags)
+ throws IOException;
+
/**
* Writes a key in an existing bucket.
* @param volumeName Name of the Volume
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index c34987090f,248f21cd70..ac0cf1d09c
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -1387,7 -1387,41 +1387,23 @@@ public class RpcClient implements Clien
ReplicationConfig replicationConfig,
Map<String, String> metadata)
throws IOException {
+ return createKey(volumeName, bucketName, keyName, size, replicationConfig,
+ metadata, Collections.emptyMap());
+ }
+
+ @Override
+ public OzoneOutputStream createKey(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags) throws IOException {
- verifyVolumeName(volumeName);
- verifyBucketName(bucketName);
- if (checkKeyNameEnabled) {
- HddsClientUtils.verifyKeyName(keyName);
- }
- HddsClientUtils.checkNotNull(keyName);
- if (omVersion
- .compareTo(OzoneManagerVersion.ERASURE_CODED_STORAGE_SUPPORT) < 0) {
- if (replicationConfig != null &&
- replicationConfig.getReplicationType()
- == HddsProtos.ReplicationType.EC) {
- throw new IOException("Can not set the replication of the key to"
- + " Erasure Coded replication, as OzoneManager does not support"
- + " Erasure Coded replication.");
- }
- }
+ createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig);
+
+ if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) {
+ if (tags != null && !tags.isEmpty()) {
+ throw new IOException("OzoneManager does not support object tags");
+ }
+ }
+
- if (replicationConfig != null) {
- replicationConfigValidator.validate(replicationConfig);
- }
String ownerName = getRealUserInfo().getShortUserName();
OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
@@@ -1758,8 -1767,8 +1804,10 @@@
keyInfo.getModificationTime(), ozoneKeyLocations,
keyInfo.getReplicationConfig(), keyInfo.getMetadata(),
keyInfo.getFileEncryptionInfo(),
- () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), keyInfo.getOwnerName(),
- keyInfo.getGeneration());
- () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(),
- keyInfo.getOwnerName(), keyInfo.getTags());
++ () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(),
++ keyInfo.getOwnerName(), keyInfo.getTags(),
++ keyInfo.getGeneration()
++ );
}
@Override
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index 6d0cbc9db0,19d5ab4fa7..ba28b45a0e
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@@ -53,13 -53,7 +53,14 @@@ public final class OmKeyArgs implement
private final boolean recursive;
private final boolean headOp;
private final boolean forceUpdateContainerCacheFromSCM;
+ private final Map<String, String> tags;
+ // expectedDataGeneration, when used in key creation indicates that a
+ // key with the same keyName should exist with the given generation.
+ // For a key commit to succeed, the original key should still be present with the
+ // generation unchanged.
+ // This allows a key to be created an committed atomically if the original has not
+ // been modified.
+ private Long expectedDataGeneration = null;
private OmKeyArgs(Builder b) {
this.volumeName = b.volumeName;
@@@ -79,7 -73,7 +80,8 @@@
this.headOp = b.headOp;
this.forceUpdateContainerCacheFromSCM = b.forceUpdateContainerCacheFromSCM;
this.ownerName = b.ownerName;
+ this.tags = b.tags;
+ this.expectedDataGeneration = b.expectedDataGeneration;
}
public boolean getIsMultipartKey() {
@@@ -158,10 -152,10 +160,14 @@@
return forceUpdateContainerCacheFromSCM;
}
+ public Map<String, String> getTags() {
+ return tags;
+ }
+
+ public Long getExpectedDataGeneration() {
+ return expectedDataGeneration;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@@ -201,12 -195,8 +207,13 @@@
.setHeadOp(headOp)
.setLatestVersionLocation(latestVersionLocation)
.setAcls(acls)
- .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCacheFromSCM);
+ .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCacheFromSCM)
+ .addAllTags(tags);
+
+ if (expectedDataGeneration != null) {
+ builder.setExpectedDataGeneration(expectedDataGeneration);
+ }
+ return builder;
}
@Nonnull
@@@ -248,7 -235,7 +255,8 @@@
private boolean recursive;
private boolean headOp;
private boolean forceUpdateContainerCacheFromSCM;
+ private final Map<String, String> tags = new HashMap<>();
+ private Long expectedDataGeneration = null;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index a9a0fb82a4,c8e7f8f609..f52a142239
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@@ -102,14 -102,11 +102,19 @@@ public final class OmKeyInfo extends Wi
*/
private final CopyOnWriteArrayList<OzoneAcl> acls;
+ /**
+ * Used for S3 tags.
+ */
+ private Map<String, String> tags;
+
+ // expectedDataGeneration, when used in key creation indicates that a
+ // key with the same keyName should exist with the given generation.
+ // For a key commit to succeed, the original key should still be present with the
+ // generation unchanged.
+ // This allows a key to be created an committed atomically if the original has not
+ // been modified.
+ private Long expectedDataGeneration = null;
+
private OmKeyInfo(Builder b) {
super(b);
this.volumeName = b.volumeName;
@@@ -126,7 -123,7 +131,8 @@@
this.fileName = b.fileName;
this.isFile = b.isFile;
this.ownerName = b.ownerName;
+ this.tags = b.tags;
+ this.expectedDataGeneration = b.expectedDataGeneration;
}
public String getVolumeName() {
@@@ -460,7 -451,7 +476,8 @@@
private FileChecksum fileChecksum;
private boolean isFile;
+ private final Map<String, String> tags = new HashMap<>();
+ private Long expectedDataGeneration = null;
public Builder() {
}
@@@ -589,11 -580,16 +606,21 @@@
return this;
}
+ public Builder addTag(String key, String value) {
+ tags.put(key, value);
+ return this;
+ }
+
+ public Builder addAllTags(Map<String, String> keyTags) {
+ tags.putAll(keyTags);
+ return this;
+ }
+
+ public Builder setExpectedDataGeneration(Long existingGeneration) {
+ this.expectedDataGeneration = existingGeneration;
+ return this;
+ }
+
public OmKeyInfo build() {
return new OmKeyInfo(this);
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
index f53fad3bdc,f3c9227126..007591ddc3
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
@@@ -187,13 -194,10 +199,10 @@@ import org.junit.jupiter.params.provide
/**
* This is an abstract class to test all the public facing APIs of Ozone
- * Client, w/o OM Ratis server.
- * {@link TestOzoneRpcClient} tests the Ozone Client by submitting the
- * requests directly to OzoneManager. {@link TestOzoneRpcClientWithRatis}
- * tests the Ozone Client by submitting requests to OM's Ratis server.
+ * Client.
*/
@TestMethodOrder(MethodOrderer.MethodName.class)
- public abstract class TestOzoneRpcClientAbstract extends OzoneTestBase {
-abstract class OzoneRpcClientTests {
++abstract class OzoneRpcClientTests extends OzoneTestBase {
private static MiniOzoneCluster cluster = null;
private static OzoneClient ozClient = null;
@@@ -4108,38 -3956,19 +4178,32 @@@
assertNotNull(omMultipartUploadCompleteInfo.getHash());
}
- private void createTestKey(OzoneBucket bucket, String keyName,
- String keyValue) throws IOException {
- OzoneOutputStream out = bucket.createKey(keyName,
- keyValue.getBytes(UTF_8).length, RATIS,
- ONE, new HashMap<>());
- out.write(keyValue.getBytes(UTF_8));
- out.close();
- OzoneKey key = bucket.getKey(keyName);
+ private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException {
+ return createTestKey(bucket, getTestName(), UUID.randomUUID().toString());
+ }
+
+ private OzoneKeyDetails createTestKey(
+ OzoneBucket bucket, String keyName, String keyValue
+ ) throws IOException {
+ return createTestKey(bucket, keyName, keyValue.getBytes(UTF_8));
+ }
+
+ private OzoneKeyDetails createTestKey(
+ OzoneBucket bucket, String keyName, byte[] bytes
+ ) throws IOException {
+ RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE);
+ Map<String, String> metadata = singletonMap("key", RandomStringUtils.randomAscii(10));
+ try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, replication, metadata)) {
+ out.write(bytes);
+ }
+ OzoneKeyDetails key = bucket.getKey(keyName);
+ assertNotNull(key);
assertEquals(keyName, key.getName());
+ return key;
}
- private void assertKeyRenamedEx(OzoneBucket bucket, String keyName)
- throws Exception {
- OMException oe = null;
- try {
- bucket.getKey(keyName);
- } catch (OMException e) {
- oe = e;
- }
+ private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) {
+ OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName));
assertEquals(KEY_NOT_FOUND, oe.getResult());
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index ccd8682c62,5f5a50d1c8..8640343904
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -1026,13 -1026,9 +1026,17 @@@ message KeyArgs
// Force OM to update container cache location from SCL
optional bool forceUpdateContainerCacheFromSCM = 20;
optional string ownerName = 21;
+
+ // S3 object tags support
+ repeated hadoop.hdds.KeyValue tags = 22;
++
+ // expectedDataGeneration, when used in key creation indicates that a
+ // key with the same keyName should exist with the given generation.
+ // For a key commit to succeed, the original key should still be present with the
+ // generation unchanged.
+ // This allows a key to be created an committed atomically if the original has not
+ // been modified.
- optional uint64 expectedDataGeneration = 22;
++ optional uint64 expectedDataGeneration = 23;
}
message KeyLocation {
@@@ -1116,13 -1112,7 +1120,14 @@@ message KeyInfo
optional FileChecksumProto fileChecksum = 18;
optional bool isFile = 19;
optional string ownerName = 20;
+ repeated hadoop.hdds.KeyValue tags = 21;
+ // expectedDataGeneration, when used in key creation indicates that a
+ // key with the same keyName should exist with the given generation.
+ // For a key commit to succeed, the original key should still be present with the
+ // generation unchanged.
+ // This allows a key to be created an committed atomically if the original has not
+ // been modified.
- optional uint64 expectedDataGeneration = 21;
++ optional uint64 expectedDataGeneration = 22;
}
message BasicKeyInfo {
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 3f9f23d4b8,50bb1053be..a90d7a28ff
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@@ -779,9 -780,12 +779,16 @@@ public abstract class OMKeyRequest exte
dbKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf(
keyArgs.getMetadataList()));
+ // Construct a new tags from KeyArgs
+ // Clear the old one when the key is overwritten
+ dbKeyInfo.getTags().clear();
+ dbKeyInfo.getTags().putAll(KeyValueUtil.getFromProtobuf(
+ keyArgs.getTagsList()));
+
+ if (keyArgs.hasExpectedDataGeneration()) {
+ dbKeyInfo.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration());
+ }
++
dbKeyInfo.setFileEncryptionInfo(encInfo);
return dbKeyInfo;
}
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index 1bc141a22e,166edb552c..4bfdd33329
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@@ -61,6 -63,6 +62,8 @@@ import org.apache.hadoop.ozone.protocol
.OMRequest;
import org.junit.jupiter.params.provider.ValueSource;
++import static java.util.Collections.emptyList;
++import static java.util.Collections.emptyMap;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
@@@ -142,8 -143,12 +145,12 @@@ public class TestOMKeyCreateRequest ext
when(ozoneManager.getOzoneLockProvider()).thenReturn(
new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
+ Map<String, String> tags = new HashMap<>();
+ tags.put("tag-key1", "tag-value1");
+ tags.put("tag-key2", "tag-value2");
+
OMRequest modifiedOmRequest =
- doPreExecute(createKeyRequest(false, 0));
- doPreExecute(createKeyRequest(false, 0, Collections.emptyMap(), tags));
++ doPreExecute(createKeyRequest(false, 0, emptyMap(), tags));
OMKeyCreateRequest omKeyCreateRequest =
getOMKeyCreateRequest(modifiedOmRequest);
@@@ -186,9 -191,13 +193,13 @@@
.getOpenKeyTable(omKeyCreateRequest.getBucketLayout())
.get(openKey));
+ tags.remove("tag-key1");
+ tags.remove("tag-key2");
+ tags.put("tag-key3", "tag-value3");
+
// Override same key again
modifiedOmRequest =
- doPreExecute(createKeyRequest(false, 0));
- doPreExecute(createKeyRequest(false, 0, Collections.emptyMap(), tags));
++ doPreExecute(createKeyRequest(false, 0, emptyMap(), tags));
id = modifiedOmRequest.getCreateKeyRequest().getClientID();
openKey = getOpenKey(id);
@@@ -520,7 -534,7 +536,7 @@@
// Create the key request without any initial metadata
OMRequest createRequestWithoutMetadata = createKeyRequest(false, 0, keyName,
- null, Collections.emptyList()); // Passing 'null' for metadata
- null); // Passing 'null' for metadata
++ null, emptyMap(), emptyList()); // Passing 'null' for metadata
OMKeyCreateRequest createOmKeyCreateRequest =
new OMKeyCreateRequest(createRequestWithoutMetadata, getBucketLayout());
@@@ -543,7 -557,7 +559,7 @@@
// Overwrite the previously created key with new metadata
OMRequest overwriteRequestWithMetadata =
- createKeyRequest(false, 0, keyName, overwriteMetadata, Collections.emptyList());
- createKeyRequest(false, 0, keyName, overwriteMetadata);
++ createKeyRequest(false, 0, keyName, overwriteMetadata, emptyMap(), emptyList());
OMKeyCreateRequest overwriteOmKeyCreateRequest =
new OMKeyCreateRequest(overwriteRequestWithMetadata, getBucketLayout());
@@@ -643,12 -657,23 +659,23 @@@
@SuppressWarnings("parameterNumber")
protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber) {
- return createKeyRequest(isMultipartKey, partNumber, keyName);
- return createKeyRequest(isMultipartKey, partNumber, Collections.emptyMap(), Collections.emptyMap());
++ return createKeyRequest(isMultipartKey, partNumber, emptyMap(), emptyMap());
+ }
+
+ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber,
+ Map<String, String> metadata, Map<String, String> tags) {
- return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, tags);
++ return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, tags, emptyList());
}
private OMRequest createKeyRequest(boolean isMultipartKey, int partNumber,
String keyName) {
- return createKeyRequest(isMultipartKey, partNumber, keyName, null, Collections.emptyList());
- return createKeyRequest(isMultipartKey, partNumber, keyName, null);
++ return createKeyRequest(isMultipartKey, partNumber, keyName, emptyMap());
+ }
+
+ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber,
+ String keyName,
+ Map<String, String> metadata) {
- return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, null);
++ return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, emptyMap(), emptyList());
}
/**
@@@ -666,7 -693,7 +695,8 @@@
protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber,
String keyName,
Map<String, String> metadata,
- Map<String, String> tags) {
++ Map<String, String> tags,
+ List<OzoneAcl> acls) {
KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
@@@ -1045,13 -991,9 +1079,13 @@@
assertEquals(NOT_A_FILE, omClientResponse.getOMResponse().getStatus());
}
-
private void createAndCheck(String keyName) throws Exception {
- createAndCheck(keyName, Collections.emptyMap(), Collections.emptyList());
- OMRequest omRequest = createKeyRequest(false, 0, keyName);
++ createAndCheck(keyName, emptyMap(), emptyList());
+ }
+
+ private OmKeyInfo createAndCheck(String keyName, Map<String, String> metadata, List<OzoneAcl> acls)
+ throws Exception {
- OMRequest omRequest = createKeyRequest(false, 0, keyName, metadata, acls);
++ OMRequest omRequest = createKeyRequest(false, 0, keyName, metadata, emptyMap(), acls);
OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest);
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index d7d092c0e5,b75167c89c..bea831a065
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@@ -227,14 -230,16 +230,24 @@@ public class ClientProtocolStub impleme
.createKey(keyName, size, replicationConfig, metadata);
}
+ @Override
+ public OzoneOutputStream createKey(String volumeName, String bucketName,
+ String keyName, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .createKey(keyName, size, replicationConfig, metadata, tags);
+ }
+
+ @Override
+ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName,
+ long size, long existingKeyGeneration, ReplicationConfig replicationConfig,
+ Map<String, String> metadata) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata);
+ }
+
@Override
public OzoneInputStream getKey(String volumeName, String bucketName,
String keyName) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org