You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/05/24 07:34:30 UTC
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into HDDS-4944
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 79a812af1f1156aed729e82a85b9492c00ef573f
Merge: 35043683a6 c01fc7907a
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Mon May 23 12:30:16 2022 -0700
Merge remote-tracking branch 'asf/master' into HDDS-4944
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
Change-Id: I05134acea37376d41d37bced6e40c700fbd9a0bf
.github/workflows/post-commit.yml | 56 +-
dev-support/annotations/pom.xml | 4 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 201 ++
.../hdds/scm/storage/ExtendedInputStream.java | 91 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 +
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 +
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 ++
.../io/ECBlockReconstructedStripeInputStream.java | 632 +++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io}/package-info.java | 6 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 +
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++
.../client/io/TestECBlockInputStreamProxy.java | 396 +++
.../io/TestECBlockReconstructedInputStream.java | 377 +++
.../TestECBlockReconstructedStripeInputStream.java | 721 ++++++
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 20 +
.../hdds/client/DefaultReplicationConfig.java | 134 +
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 20 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../hadoop/hdds/freon/FakeClusterTopology.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 66 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 -
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 +
.../common/src/main/resources/ozone-default.xml | 111 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 +
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +-
.../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 +
.../hdds/scm/container/TestContainerInfo.java | 86 +
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 +
.../container/common/helpers/TestExcludeList.java | 67 +
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +-
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +-
.../common/src/test/resources/auditlog.properties | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/report/ContainerReportPublisher.java | 1 +
.../common/report/PipelineReportPublisher.java | 1 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 78 +
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../ECReconstructionCommandInfo.java | 82 +
.../ECReconstructionCoordinatorTask.java | 52 +
.../reconstruction/ECReconstructionSupervisor.java | 72 +
.../container/ec/reconstruction/package-info.java | 34 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 1 +
.../commands/ReconstructECContainersCommand.java | 193 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../TestECReconstructionSupervisor.java | 53 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../docs/content/security/SecurityWithRanger.md | 7 +
.../docs/content/security/SecurityWithRanger.zh.md | 9 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 -
hadoop-hdds/docs/pom.xml | 12 +-
.../_markup/render-image.html} | 3 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 +
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 +
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 +
.../rawcoder/AbstractNativeRawEncoder.java | 99 +
.../rawcoder/ByteArrayDecodingState.java | 132 +
.../rawcoder/ByteArrayEncodingState.java | 105 +
.../rawcoder/ByteBufferDecodingState.java | 143 ++
.../rawcoder/ByteBufferEncodingState.java | 110 +
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 +
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 +
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 +
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 +
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 +
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 +
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 +
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 +
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 +
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 +
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 +
.../ozone/erasurecode/TestCodecRegistry.java | 159 ++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 +
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 +
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++
.../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 +
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 4 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../SCMBlockLocationFailoverProxyProvider.java | 3 +
.../client/DefaultCertificateClient.java | 9 +-
.../exceptions/TestSCMExceptionResultCodes.java | 19 +-
.../token/TestOzoneBlockTokenIdentifier.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 4 +-
.../certificate/authority/TestDefaultCAServer.java | 40 +-
.../certificate/authority/TestDefaultProfile.java | 22 +-
.../client/TestCertificateClientInit.java | 73 +-
.../client/TestDefaultCertificateClient.java | 60 +-
.../x509/certificate/utils/TestCRLCodec.java | 26 +-
.../certificate/utils/TestCertificateCodec.java | 41 +-
.../certificates/TestCertificateSignRequest.java | 60 +-
.../x509/certificates/TestRootCertificate.java | 68 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 14 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 49 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 28 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 28 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hdds/server/events/TestEventWatcher.java | 46 +-
.../hdds/server/http/TestBaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +-
.../hdds/server/http/TestHttpRequestLog.java | 16 +-
.../server/http/TestHttpRequestLogAppender.java | 12 +-
.../hadoop/hdds/server/http/TestHttpServer2.java | 4 +-
.../hdds/server/http/TestProfileServlet.java | 39 +-
.../server/http/TestPrometheusMetricsSink.java | 47 +-
.../server/http/TestRatisDropwizardExports.java | 13 +-
.../hdds/server/http/TestRatisNameRewrite.java | 65 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 78 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +-
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +-
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 38 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/ECContainerReplicaCount.java | 313 +++
.../scm/container/balancer/ContainerBalancer.java | 135 +-
.../balancer/ContainerBalancerMetrics.java | 70 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 +++
.../container/replication/ContainerReplicaOp.java | 63 +
.../replication/ContainerReplicaPendingOps.java | 227 ++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 +--
.../replication/ReplicationActivityStatus.java | 104 -
.../container/replication/ReplicationManager.java | 453 ++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 +
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 +
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 +
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 31 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 ++
.../algorithms/RandomPipelineChoosePolicy.java | 1 +
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../scm/server/StorageContainerManagerStarter.java | 3 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 12 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +-
.../command/TestCommandStatusReportHandler.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 39 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 109 +-
.../scm/container/TestContainerReportHandler.java | 207 +-
.../scm/container/TestContainerStateManager.java | 18 +-
.../TestIncrementalContainerReportHandler.java | 34 +-
.../hdds/scm/container/TestReplicationManager.java | 573 +++--
.../scm/container/TestUnknownContainerReport.java | 10 +-
.../container/balancer/TestContainerBalancer.java | 55 +-
.../container/balancer/TestFindTargetStrategy.java | 26 +-
.../algorithms/TestContainerPlacementFactory.java | 43 +-
.../TestContainerPlacementStatusDefault.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 28 +-
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +--
.../TestSCMContainerPlacementRandom.java | 36 +-
.../TestContainerReplicaPendingOps.java | 226 ++
.../replication/TestReplicationManagerMetrics.java | 27 +-
.../states/TestContainerReplicaCount.java | 12 +-
.../states/TestECContainerReplicaCount.java | 432 ++++
.../hdds/scm/ha/TestReplicationAnnotation.java | 20 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +-
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 89 +
.../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +-
...ler.java => TestCommandQueueReportHandler.java} | 106 +-
.../hdds/scm/node/TestContainerPlacement.java | 7 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 10 +-
.../hdds/scm/node/TestNodeReportHandler.java | 22 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 12 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 +
.../scm/pipeline/TestRatisPipelineProvider.java | 93 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 44 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++
.../TestPipelineChoosePolicyFactory.java | 14 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 26 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 23 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../scm/server/TestSCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +-
.../server/TestStorageContainerManagerStarter.java | 18 +-
.../ozone/container/common/TestEndPoint.java | 84 +-
.../placement/TestContainerPlacement.java | 10 +-
.../container/placement/TestDatanodeMetrics.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 21 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +-
.../org/apache/ozone/test/GenericTestUtils.java | 1 +
hadoop-hdds/tools/pom.xml | 7 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +-
.../scm/cli/container/TestReportSubCommand.java | 36 +-
.../datanode/TestContainerBalancerSubCommand.java | 14 +-
.../cli/datanode/TestDecommissionSubCommand.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 14 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 16 +-
.../cli/datanode/TestRecommissionSubCommand.java | 14 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 +
.../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 +
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 +-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 ++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +-
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++
.../ozone/om/ha/OMFailoverProxyProvider.java | 22 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++
.../common/src/test/resources/log4j.properties | 21 +
hadoop-ozone/csi/pom.xml | 17 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 11 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 12 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 +
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../buckethead.robot => basic/single_node.robot} | 22 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 +
.../dist/src/main/smoketest/ec/basic.robot | 64 +
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 1 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 1 +
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/s3/webui.robot | 1 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 2 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
hadoop-ozone/insight/pom.xml | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 9 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++--
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +-
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 +
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 44 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
hadoop-ozone/ozone-manager/pom.xml | 13 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 10 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 316 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 +
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 +
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 57 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 67 +-
.../request/file/OMFileCreateRequestWithFSO.java | 19 +-
.../om/request/key/OMAllocateBlockRequest.java | 52 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 72 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 62 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 26 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCompleteRequest.java | 60 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/request/validation/ValidationContext.java | 24 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 14 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 5 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 13 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 107 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 -
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 99 +
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 55 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadCompleteRequest.java | 8 +
.../request/validation/TestRequestValidations.java | 95 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 59 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/pom.xml | 12 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 8 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++
.../hadoop/ozone/s3/OzoneClientProducer.java | 41 +-
.../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 +
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++
hadoop-ozone/tools/pom.xml | 2 +-
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 +
.../hadoop/ozone/freon/ContentGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../ozone/freon/FreonReplicationOptions.java | 73 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 +
.../ozone/freon/HadoopNestedDirGenerator.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 108 +
.../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 +
.../containergenerator/GeneratorDatanode.java | 1 +
.../hadoop/ozone/shell/ReplicationOptions.java | 87 +
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 87 +-
685 files changed, 40052 insertions(+), 5758 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f0a26df46f,f1c3d6578b..067777bb79
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -473,55 -455,6 +474,58 @@@ public final class OzoneConsts
public static final String DELEGATION_TOKEN_SERVICE = "service";
public static final String DELEGATION_TOKEN_RENEWER = "renewer";
+ // EC Constants
+ public static final String BLOCK_GROUP_LEN_KEY_IN_PUT_BLOCK = "blockGroupLen";
++
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users";
+
+ // Ideally we should use /addUsersAndGroups endpoint for add user to role,
+ // but it always return 405 somehow.
+ // https://ranger.apache.org/apidocs/resource_RoleREST.html
+ // #resource_RoleREST_addUsersAndGroups_PUT
+ public static final String OZONE_OM_RANGER_ADMIN_ROLE_ADD_USER_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_USER_HTTP_ENDPOINT =
+ "/service/xusers/users/?name=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/name/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_GROUP_HTTP_ENDPOINT =
+ "/service/xusers/secure/groups/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyName=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_ID_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyId=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_POLICY_HTTP_ENDPOINT =
+ "/service/plugins/policies/";
+
+ public static final String OZONE_OM_RANGER_OZONE_SERVICE_ENDPOINT =
+ "/service/plugins/services/";
+
+ public static final String OZONE_OM_RANGER_DOWNLOAD_ENDPOINT =
+ "/service/plugins/secure/policies/download/cm_ozone" +
+ "?supportsPolicyDeltas=true&lastKnownVersion=";
+
+ public static final String OZONE_OM_RANGER_ALL_POLICIES_ENDPOINT =
+ "/service/plugins/policies/service/";
+
+ public static final String OZONE_TENANT_RANGER_POLICY_LABEL = "OzoneTenant";
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3077bc73c1,c7764e9d5b..db0882c80b
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -77,8 -86,10 +86,11 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.TenantArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
+ import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 291eb6f28d,bc1ed8d6f5..aa34f049f4
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -312,44 -341,11 +341,52 @@@ public final class OMConfigKeys
public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
= 10000;
+ /**
+ * This configuration shall be enabled to utilize the functionality of the
+ * fine-grained KEY_PATH_LOCK.
+ */
+ public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED =
+ "ozone.om.key.path.lock.enabled";
+ public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false;
++
+ public static final String OZONE_OM_MULTITENANCY_ENABLED =
+ "ozone.om.multitenancy.enabled";
+ public static final boolean OZONE_OM_MULTITENANCY_ENABLED_DEFAULT = false;
+
+ /**
+ * Temporary configuration properties for Ranger REST use in multitenancy.
+ */
+ public static final String OZONE_RANGER_OM_IGNORE_SERVER_CERT =
+ "ozone.om.ranger.ignore.cert";
+ public static final boolean OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT =
+ true;
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT =
+ "ozone.om.ranger.connection.timeout";
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT =
+ "ozone.om.ranger.connection.request.timeout";
+ public static final String
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_USER =
+ "ozone.om.ranger.https.admin.api.user";
+ // TODO: Note this should be removed once Ranger Java Client is in place.
+ // And Ranger SPNEGO auth (ranger.spnego.kerberos.principal ?) should be used
+ // instead. Or keep this solely for dev testing. See HDDS-5836.
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD =
+ "ozone.om.ranger.https.admin.api.passwd";
+ public static final String OZONE_RANGER_HTTPS_ADDRESS_KEY =
+ "ozone.om.ranger.https-address";
+ public static final String OZONE_RANGER_SERVICE =
+ "ozone.om.ranger.service";
+
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL
+ = "ozone.om.multitenancy.ranger.sync.interval";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL_DEFAULT
+ = TimeDuration.valueOf(600, TimeUnit.SECONDS);
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT
+ = "ozone.om.multitenancy.ranger.sync.timeout";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT_DEFAULT
+ = TimeDuration.valueOf(10, TimeUnit.SECONDS);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 717df88b48,168b2c8bb4..a669f8a7df
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -243,19 -243,6 +243,20 @@@ public class OMException extends IOExce
LAYOUT_FEATURE_FINALIZATION_FAILED,
PREPARE_FAILED,
NOT_SUPPORTED_OPERATION_WHEN_PREPARED,
- NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION
++ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+
+ TENANT_NOT_FOUND,
+ TENANT_ALREADY_EXISTS, INVALID_TENANT_ID,
+
+ ACCESS_ID_NOT_FOUND,
+ TENANT_USER_ACCESS_ID_ALREADY_EXISTS,
+ INVALID_TENANT_USERNAME,
+ INVALID_ACCESS_ID,
+ TENANT_AUTHORIZER_ERROR,
+
+ VOLUME_IS_REFERENCED,
+ TENANT_NOT_EMPTY,
+
+ FEATURE_NOT_ENABLED
}
}
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
index 0000000000,d54c9bc89b..f46f067914
mode 000000,100755..100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
@@@ -1,0 -1,84 +1,84 @@@
+ #!/usr/bin/env bash
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ source "$TEST_DIR"/testlib.sh
+
+ # Helper function, not a callback.
+ _check_hdds_mlvs() {
+ mlv="$1"
+ check_scm_mlv scm "$mlv"
+ check_dn_mlv dn1 "$mlv"
+ check_dn_mlv dn2 "$mlv"
+ check_dn_mlv dn3 "$mlv"
+ }
+
+ # Helper function, not a callback.
+ _check_om_mlvs() {
+ mlv="$1"
+ check_om_mlv om1 "$mlv"
+ check_om_mlv om2 "$mlv"
+ check_om_mlv om3 "$mlv"
+ }
+
+ setup() {
+ export OZONE_OM_PREPARE='true'
+ }
+
+ with_old_version() {
+ generate old1
+ validate old1
+ }
+
+ with_new_version_pre_finalized() {
+ _check_hdds_mlvs 2
+ _check_om_mlvs 0
+
+ validate old1
+ # HDDS-6261: overwrite the same keys intentionally
+ generate old1 --exclude create-volume-and-bucket
+
+ generate new1
+ validate new1
+
+ check_ec_is_disabled
+ }
+
+ with_old_version_downgraded() {
+ validate old1
+ validate new1
+
+ generate old2
+ validate old2
+
+ # HDDS-6261: overwrite the same keys again to trigger the precondition check
+ # that exists <= 1.1.0 OM
+ generate old1 --exclude create-volume-and-bucket
+ }
+
+ with_new_version_finalized() {
+ _check_hdds_mlvs 3
- _check_om_mlvs 2
++ _check_om_mlvs 3
+
+ validate old1
+ validate new1
+ validate old2
+
+ generate new2
+ validate new2
+
+ check_ec_is_enabled
+ }
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
index 65aae1e7ed,21edfe5d15..ad025d19f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
@@@ -26,8 -26,8 +26,7 @@@ import static org.apache.hadoop.hdds.pr
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
- import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 15eda5a4b9,3b9d3df975..f9245268f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -113,13 -116,9 +120,15 @@@ public class TestOzoneConfigurationFiel
ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_FLUSH_PARAM,
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
OMConfigKeys.OZONE_OM_HA_PREFIX,
+ OMConfigKeys.OZONE_OM_TRANSPORT_CLASS,
- OMConfigKeys.OZONE_OM_GRPC_PORT_KEY
++ OMConfigKeys.OZONE_OM_GRPC_PORT_KEY,
// TODO HDDS-2856
+ OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_HTTPS_ADDRESS_KEY,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD
));
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index fdbf17d9a4,e38c413e74..69977648a1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@@ -102,9 -108,8 +109,9 @@@ public class TestOzoneShellHA
private static String testFilePathString;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
- private static OzoneShell ozoneShell = null;
- private static OzoneAdmin ozoneAdminShell = null;
- private static S3Shell s3Shell = null;
+ private OzoneShell ozoneShell = null;
+ private OzoneAdmin ozoneAdminShell = null;
++ private S3Shell s3Shell = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final ByteArrayOutputStream err = new ByteArrayOutputStream();
@@@ -170,6 -173,8 +175,9 @@@
@Before
public void setup() throws UnsupportedEncodingException {
+ ozoneShell = new OzoneShell();
+ ozoneAdminShell = new OzoneAdmin();
++ s3Shell = new S3Shell();
System.setOut(new PrintStream(out, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(err, false, DEFAULT_ENCODING));
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index deb77db639,d91d8519a9..c6a6745d5f
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -411,21 -356,7 +411,22 @@@ enum Status
LAYOUT_FEATURE_FINALIZATION_FAILED = 72;
PREPARE_FAILED = 73;
NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74;
+ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION = 75;
+
- TENANT_NOT_FOUND = 75;
- TENANT_ALREADY_EXISTS = 76;
- INVALID_TENANT_ID = 77;
++ TENANT_NOT_FOUND = 76;
++ TENANT_ALREADY_EXISTS = 77;
++ INVALID_TENANT_ID = 78;
+
- ACCESS_ID_NOT_FOUND = 78;
- TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 79;
- INVALID_TENANT_USERNAME = 80;
- INVALID_ACCESS_ID = 81;
- TENANT_AUTHORIZER_ERROR = 82;
++ ACCESS_ID_NOT_FOUND = 79;
++ TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 80;
++ INVALID_TENANT_USERNAME = 81;
++ INVALID_ACCESS_ID = 82;
++ TENANT_AUTHORIZER_ERROR = 83;
+
- VOLUME_IS_REFERENCED = 83;
- TENANT_NOT_EMPTY = 84;
++ VOLUME_IS_REFERENCED = 84;
++ TENANT_NOT_EMPTY = 85;
+
- FEATURE_NOT_ENABLED = 85;
++ FEATURE_NOT_ENABLED = 86;
}
/**
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d2848748a1,4e051875d7..ef2e8d70e9
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -67,10 -69,8 +69,11 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneManagerVersion;
+ import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
@@@ -127,12 -124,9 +130,13 @@@ import org.apache.hadoop.ozone.om.helpe
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.TenantStateList;
+import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
+import org.apache.hadoop.ozone.om.helpers.TenantUserList;
+ import org.apache.hadoop.ozone.om.lock.OzoneLockProvider;
import org.apache.hadoop.ozone.om.protocol.OMInterServiceProtocol;
import org.apache.hadoop.ozone.om.protocol.OMConfiguration;
import org.apache.hadoop.ozone.om.protocolPB.OMInterServiceProtocolClientSideImpl;
@@@ -300,6 -301,7 +314,7 @@@ public final class OzoneManager extend
private final Text omRpcAddressTxt;
private OzoneConfiguration configuration;
private RPC.Server omRpcServer;
- private GrpcOzoneManagerServer omS3gGrpcServer;
++ private GrpcOzoneManagerServer omS3gGrpcServer;
private InetSocketAddress omRpcAddress;
private String omId;
@@@ -339,7 -340,9 +354,9 @@@
private JvmPauseMonitor jvmPauseMonitor;
private final SecurityConfig secConfig;
private S3SecretManager s3SecretManager;
+ private final boolean isOmGrpcServerEnabled;
private volatile boolean isOmRpcServerRunning = false;
- private volatile boolean isOmGrpcServerRunning = false;
++ private volatile boolean isOmGrpcServerRunning = false;
private String omComponent;
private OzoneManagerProtocolServerSideTranslatorPB omServerProtocol;
@@@ -3910,21 -3741,20 +4057,21 @@@
throws IOException {
Pair<String, String> resolved;
- try {
- if (isAclEnabled) {
- InetAddress remoteIp = Server.getRemoteIp();
- resolved = resolveBucketLink(requested, new HashSet<>(),
- getRemoteUser(),
- remoteIp,
- remoteIp != null ? remoteIp.getHostName() :
- omRpcAddress.getHostName());
- } else {
- resolved = resolveBucketLink(requested, new HashSet<>(),
- null, null, null);
+ if (isAclEnabled) {
- UserGroupInformation ugi = Server.getRemoteUser();
++ UserGroupInformation ugi = getRemoteUser();
+ if (getS3Auth() != null) {
+ ugi = UserGroupInformation.createRemoteUser(
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId()));
}
- } catch (Throwable t) {
- throw t;
+ InetAddress remoteIp = Server.getRemoteIp();
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ ugi,
+ remoteIp,
+ remoteIp != null ? remoteIp.getHostName() :
+ omRpcAddress.getHostName());
+ } else {
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ null, null, null);
}
return new ResolvedBucket(requested, resolved);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 60a888e446,7b1f4185b6..4302327cf3
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@@ -31,8 -31,13 +31,14 @@@ public enum OMLayoutFeature implements
////////////////////////////// //////////////////////////////
INITIAL_VERSION(0, "Initial Layout Version"),
- // TODO: Make this 2 after bringing in EC
- MULTITENANCY_SCHEMA(1, "Multi-Tenancy Schema");
+ ERASURE_CODED_STORAGE_SUPPORT(1, "Ozone version with built in support for"
+ + " Erasure Coded block data storage."),
+
+ BUCKET_LAYOUT_SUPPORT(2, "Ozone version supporting bucket " +
+ "layouts and introducing the FILE_SYSTEM_OPTIMIZED and OBJECT_STORE " +
- "bucket layout types.");
++ "bucket layout types."),
+
++ MULTITENANCY_SCHEMA(3, "Multi-Tenancy Schema");
/////////////////////////////// /////////////////////////////
// Example OM Layout Feature with Actions
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 4dffc39e22,ecec94592d..94b9b6d56c
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -48,8 -48,11 +51,12 @@@ import org.apache.hadoop.ozone.om.ratis
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+ import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+ import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
@@@ -78,15 -80,12 +86,17 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index 891c48ca96,e9ddc08815..ca8d32b48d
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@@ -71,9 -66,7 +66,7 @@@ public class OzoneClientProducer
@Produces
public synchronized OzoneClient createClient() throws WebApplicationException,
IOException {
- if (client == null) {
- client = createOzoneClient();
- }
- client = getClient(ozoneConfiguration);
++ client = getClient(ozoneConfiguration);
return client;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org