You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/05/24 08:55:27 UTC
[ozone] branch HDDS-4944 updated (79a812af1f -> 92daac20f0)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 79a812af1f Merge remote-tracking branch 'asf/master' into HDDS-4944
new 92daac20f0 Merge remote-tracking branch 'asf/master' into HDDS-4944
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (79a812af1f)
\
N -- N -- N refs/heads/HDDS-4944 (92daac20f0)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into HDDS-4944
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 92daac20f0375ecaff2f6b5aa5073c5f99f13d89
Merge: 35043683a6 c01fc7907a
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Mon May 23 12:30:16 2022 -0700
Merge remote-tracking branch 'asf/master' into HDDS-4944
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
Change-Id: I05134acea37376d41d37bced6e40c700fbd9a0bf
.github/workflows/post-commit.yml | 56 +-
dev-support/annotations/pom.xml | 4 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 201 ++
.../hdds/scm/storage/ExtendedInputStream.java | 91 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 +
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 +
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 ++
.../io/ECBlockReconstructedStripeInputStream.java | 632 +++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io}/package-info.java | 6 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 +
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++
.../client/io/TestECBlockInputStreamProxy.java | 396 +++
.../io/TestECBlockReconstructedInputStream.java | 377 +++
.../TestECBlockReconstructedStripeInputStream.java | 721 ++++++
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 20 +
.../hdds/client/DefaultReplicationConfig.java | 134 +
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 20 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../hadoop/hdds/freon/FakeClusterTopology.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 66 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 -
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 +
.../common/src/main/resources/ozone-default.xml | 111 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 +
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +-
.../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 +
.../hdds/scm/container/TestContainerInfo.java | 86 +
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 +
.../container/common/helpers/TestExcludeList.java | 67 +
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +-
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +-
.../common/src/test/resources/auditlog.properties | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/report/ContainerReportPublisher.java | 1 +
.../common/report/PipelineReportPublisher.java | 1 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 78 +
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../ECReconstructionCommandInfo.java | 82 +
.../ECReconstructionCoordinatorTask.java | 52 +
.../reconstruction/ECReconstructionSupervisor.java | 72 +
.../container/ec/reconstruction/package-info.java | 34 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 1 +
.../commands/ReconstructECContainersCommand.java | 193 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../TestECReconstructionSupervisor.java | 53 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../docs/content/security/SecurityWithRanger.md | 7 +
.../docs/content/security/SecurityWithRanger.zh.md | 9 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 -
hadoop-hdds/docs/pom.xml | 12 +-
.../_markup/render-image.html} | 3 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 +
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 +
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 +
.../rawcoder/AbstractNativeRawEncoder.java | 99 +
.../rawcoder/ByteArrayDecodingState.java | 132 +
.../rawcoder/ByteArrayEncodingState.java | 105 +
.../rawcoder/ByteBufferDecodingState.java | 143 ++
.../rawcoder/ByteBufferEncodingState.java | 110 +
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 +
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 +
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 +
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 +
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 +
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 +
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 +
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 +
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 +
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 +
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 +
.../ozone/erasurecode/TestCodecRegistry.java | 159 ++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 +
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 +
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++
.../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 +
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 4 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../SCMBlockLocationFailoverProxyProvider.java | 3 +
.../client/DefaultCertificateClient.java | 9 +-
.../exceptions/TestSCMExceptionResultCodes.java | 19 +-
.../token/TestOzoneBlockTokenIdentifier.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 4 +-
.../certificate/authority/TestDefaultCAServer.java | 40 +-
.../certificate/authority/TestDefaultProfile.java | 22 +-
.../client/TestCertificateClientInit.java | 73 +-
.../client/TestDefaultCertificateClient.java | 60 +-
.../x509/certificate/utils/TestCRLCodec.java | 26 +-
.../certificate/utils/TestCertificateCodec.java | 41 +-
.../certificates/TestCertificateSignRequest.java | 60 +-
.../x509/certificates/TestRootCertificate.java | 68 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 14 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 49 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 28 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 28 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hdds/server/events/TestEventWatcher.java | 46 +-
.../hdds/server/http/TestBaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +-
.../hdds/server/http/TestHttpRequestLog.java | 16 +-
.../server/http/TestHttpRequestLogAppender.java | 12 +-
.../hadoop/hdds/server/http/TestHttpServer2.java | 4 +-
.../hdds/server/http/TestProfileServlet.java | 39 +-
.../server/http/TestPrometheusMetricsSink.java | 47 +-
.../server/http/TestRatisDropwizardExports.java | 13 +-
.../hdds/server/http/TestRatisNameRewrite.java | 65 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 78 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +-
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +-
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 38 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/ECContainerReplicaCount.java | 313 +++
.../scm/container/balancer/ContainerBalancer.java | 135 +-
.../balancer/ContainerBalancerMetrics.java | 70 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 +++
.../container/replication/ContainerReplicaOp.java | 63 +
.../replication/ContainerReplicaPendingOps.java | 227 ++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 +--
.../replication/ReplicationActivityStatus.java | 104 -
.../container/replication/ReplicationManager.java | 453 ++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 +
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 +
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 +
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 31 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 ++
.../algorithms/RandomPipelineChoosePolicy.java | 1 +
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../scm/server/StorageContainerManagerStarter.java | 3 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 12 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +-
.../command/TestCommandStatusReportHandler.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 39 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 109 +-
.../scm/container/TestContainerReportHandler.java | 207 +-
.../scm/container/TestContainerStateManager.java | 18 +-
.../TestIncrementalContainerReportHandler.java | 34 +-
.../hdds/scm/container/TestReplicationManager.java | 573 +++--
.../scm/container/TestUnknownContainerReport.java | 10 +-
.../container/balancer/TestContainerBalancer.java | 55 +-
.../container/balancer/TestFindTargetStrategy.java | 26 +-
.../algorithms/TestContainerPlacementFactory.java | 43 +-
.../TestContainerPlacementStatusDefault.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 28 +-
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +--
.../TestSCMContainerPlacementRandom.java | 36 +-
.../TestContainerReplicaPendingOps.java | 226 ++
.../replication/TestReplicationManagerMetrics.java | 27 +-
.../states/TestContainerReplicaCount.java | 12 +-
.../states/TestECContainerReplicaCount.java | 432 ++++
.../hdds/scm/ha/TestReplicationAnnotation.java | 20 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +-
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 89 +
.../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +-
...ler.java => TestCommandQueueReportHandler.java} | 106 +-
.../hdds/scm/node/TestContainerPlacement.java | 7 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 10 +-
.../hdds/scm/node/TestNodeReportHandler.java | 22 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 12 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 +
.../scm/pipeline/TestRatisPipelineProvider.java | 93 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 44 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++
.../TestPipelineChoosePolicyFactory.java | 14 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 26 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 23 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../scm/server/TestSCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +-
.../server/TestStorageContainerManagerStarter.java | 18 +-
.../ozone/container/common/TestEndPoint.java | 84 +-
.../placement/TestContainerPlacement.java | 10 +-
.../container/placement/TestDatanodeMetrics.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 21 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +-
.../org/apache/ozone/test/GenericTestUtils.java | 1 +
hadoop-hdds/tools/pom.xml | 7 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +-
.../scm/cli/container/TestReportSubCommand.java | 36 +-
.../datanode/TestContainerBalancerSubCommand.java | 14 +-
.../cli/datanode/TestDecommissionSubCommand.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 14 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 16 +-
.../cli/datanode/TestRecommissionSubCommand.java | 14 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 +
.../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 +
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 +-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 ++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +-
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++
.../ozone/om/ha/OMFailoverProxyProvider.java | 22 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++
.../common/src/test/resources/log4j.properties | 21 +
hadoop-ozone/csi/pom.xml | 17 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 11 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 12 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 +
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../buckethead.robot => basic/single_node.robot} | 22 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 +
.../dist/src/main/smoketest/ec/basic.robot | 64 +
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 1 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 1 +
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/s3/webui.robot | 1 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 2 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
hadoop-ozone/insight/pom.xml | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 9 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++--
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +-
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 +
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 44 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
hadoop-ozone/ozone-manager/pom.xml | 13 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 10 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 329 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 +
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 +
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 57 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 67 +-
.../request/file/OMFileCreateRequestWithFSO.java | 19 +-
.../om/request/key/OMAllocateBlockRequest.java | 52 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 72 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 62 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 26 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCompleteRequest.java | 60 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/request/validation/ValidationContext.java | 24 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 14 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 5 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 13 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 107 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 -
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 99 +
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 55 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadCompleteRequest.java | 8 +
.../request/validation/TestRequestValidations.java | 95 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 59 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/pom.xml | 12 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 8 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++
.../hadoop/ozone/s3/OzoneClientProducer.java | 41 +-
.../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 +
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++
hadoop-ozone/tools/pom.xml | 2 +-
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 +
.../hadoop/ozone/freon/ContentGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../ozone/freon/FreonReplicationOptions.java | 73 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 +
.../ozone/freon/HadoopNestedDirGenerator.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 108 +
.../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 +
.../containergenerator/GeneratorDatanode.java | 1 +
.../hadoop/ozone/shell/ReplicationOptions.java | 87 +
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 87 +-
685 files changed, 40055 insertions(+), 5768 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f0a26df46f,f1c3d6578b..067777bb79
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -473,55 -455,6 +474,58 @@@ public final class OzoneConsts
public static final String DELEGATION_TOKEN_SERVICE = "service";
public static final String DELEGATION_TOKEN_RENEWER = "renewer";
+ // EC Constants
+ public static final String BLOCK_GROUP_LEN_KEY_IN_PUT_BLOCK = "blockGroupLen";
++
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users";
+
+ // Ideally we should use /addUsersAndGroups endpoint for add user to role,
+ // but it always return 405 somehow.
+ // https://ranger.apache.org/apidocs/resource_RoleREST.html
+ // #resource_RoleREST_addUsersAndGroups_PUT
+ public static final String OZONE_OM_RANGER_ADMIN_ROLE_ADD_USER_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_USER_HTTP_ENDPOINT =
+ "/service/xusers/users/?name=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/name/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_GROUP_HTTP_ENDPOINT =
+ "/service/xusers/secure/groups/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyName=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_ID_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyId=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_POLICY_HTTP_ENDPOINT =
+ "/service/plugins/policies/";
+
+ public static final String OZONE_OM_RANGER_OZONE_SERVICE_ENDPOINT =
+ "/service/plugins/services/";
+
+ public static final String OZONE_OM_RANGER_DOWNLOAD_ENDPOINT =
+ "/service/plugins/secure/policies/download/cm_ozone" +
+ "?supportsPolicyDeltas=true&lastKnownVersion=";
+
+ public static final String OZONE_OM_RANGER_ALL_POLICIES_ENDPOINT =
+ "/service/plugins/policies/service/";
+
+ public static final String OZONE_TENANT_RANGER_POLICY_LABEL = "OzoneTenant";
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3077bc73c1,c7764e9d5b..db0882c80b
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -77,8 -86,10 +86,11 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.TenantArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
+ import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 291eb6f28d,bc1ed8d6f5..aa34f049f4
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -312,44 -341,11 +341,52 @@@ public final class OMConfigKeys
public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
= 10000;
+ /**
+ * This configuration shall be enabled to utilize the functionality of the
+ * fine-grained KEY_PATH_LOCK.
+ */
+ public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED =
+ "ozone.om.key.path.lock.enabled";
+ public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false;
++
+ public static final String OZONE_OM_MULTITENANCY_ENABLED =
+ "ozone.om.multitenancy.enabled";
+ public static final boolean OZONE_OM_MULTITENANCY_ENABLED_DEFAULT = false;
+
+ /**
+ * Temporary configuration properties for Ranger REST use in multitenancy.
+ */
+ public static final String OZONE_RANGER_OM_IGNORE_SERVER_CERT =
+ "ozone.om.ranger.ignore.cert";
+ public static final boolean OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT =
+ true;
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT =
+ "ozone.om.ranger.connection.timeout";
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT =
+ "ozone.om.ranger.connection.request.timeout";
+ public static final String
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_USER =
+ "ozone.om.ranger.https.admin.api.user";
+ // TODO: Note this should be removed once Ranger Java Client is in place.
+ // And Ranger SPNEGO auth (ranger.spnego.kerberos.principal ?) should be used
+ // instead. Or keep this solely for dev testing. See HDDS-5836.
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD =
+ "ozone.om.ranger.https.admin.api.passwd";
+ public static final String OZONE_RANGER_HTTPS_ADDRESS_KEY =
+ "ozone.om.ranger.https-address";
+ public static final String OZONE_RANGER_SERVICE =
+ "ozone.om.ranger.service";
+
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL
+ = "ozone.om.multitenancy.ranger.sync.interval";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL_DEFAULT
+ = TimeDuration.valueOf(600, TimeUnit.SECONDS);
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT
+ = "ozone.om.multitenancy.ranger.sync.timeout";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT_DEFAULT
+ = TimeDuration.valueOf(10, TimeUnit.SECONDS);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 717df88b48,168b2c8bb4..a669f8a7df
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -243,19 -243,6 +243,20 @@@ public class OMException extends IOExce
LAYOUT_FEATURE_FINALIZATION_FAILED,
PREPARE_FAILED,
NOT_SUPPORTED_OPERATION_WHEN_PREPARED,
- NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION
++ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+
+ TENANT_NOT_FOUND,
+ TENANT_ALREADY_EXISTS, INVALID_TENANT_ID,
+
+ ACCESS_ID_NOT_FOUND,
+ TENANT_USER_ACCESS_ID_ALREADY_EXISTS,
+ INVALID_TENANT_USERNAME,
+ INVALID_ACCESS_ID,
+ TENANT_AUTHORIZER_ERROR,
+
+ VOLUME_IS_REFERENCED,
+ TENANT_NOT_EMPTY,
+
+ FEATURE_NOT_ENABLED
}
}
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
index 0000000000,d54c9bc89b..f46f067914
mode 000000,100755..100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
@@@ -1,0 -1,84 +1,84 @@@
+ #!/usr/bin/env bash
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ source "$TEST_DIR"/testlib.sh
+
+ # Helper function, not a callback.
+ _check_hdds_mlvs() {
+ mlv="$1"
+ check_scm_mlv scm "$mlv"
+ check_dn_mlv dn1 "$mlv"
+ check_dn_mlv dn2 "$mlv"
+ check_dn_mlv dn3 "$mlv"
+ }
+
+ # Helper function, not a callback.
+ _check_om_mlvs() {
+ mlv="$1"
+ check_om_mlv om1 "$mlv"
+ check_om_mlv om2 "$mlv"
+ check_om_mlv om3 "$mlv"
+ }
+
+ setup() {
+ export OZONE_OM_PREPARE='true'
+ }
+
+ with_old_version() {
+ generate old1
+ validate old1
+ }
+
+ with_new_version_pre_finalized() {
+ _check_hdds_mlvs 2
+ _check_om_mlvs 0
+
+ validate old1
+ # HDDS-6261: overwrite the same keys intentionally
+ generate old1 --exclude create-volume-and-bucket
+
+ generate new1
+ validate new1
+
+ check_ec_is_disabled
+ }
+
+ with_old_version_downgraded() {
+ validate old1
+ validate new1
+
+ generate old2
+ validate old2
+
+ # HDDS-6261: overwrite the same keys again to trigger the precondition check
+ # that exists <= 1.1.0 OM
+ generate old1 --exclude create-volume-and-bucket
+ }
+
+ with_new_version_finalized() {
+ _check_hdds_mlvs 3
- _check_om_mlvs 2
++ _check_om_mlvs 3
+
+ validate old1
+ validate new1
+ validate old2
+
+ generate new2
+ validate new2
+
+ check_ec_is_enabled
+ }
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
index 65aae1e7ed,21edfe5d15..ad025d19f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
@@@ -26,8 -26,8 +26,7 @@@ import static org.apache.hadoop.hdds.pr
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
- import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 15eda5a4b9,3b9d3df975..f9245268f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -113,13 -116,9 +120,15 @@@ public class TestOzoneConfigurationFiel
ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_FLUSH_PARAM,
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
OMConfigKeys.OZONE_OM_HA_PREFIX,
+ OMConfigKeys.OZONE_OM_TRANSPORT_CLASS,
- OMConfigKeys.OZONE_OM_GRPC_PORT_KEY
++ OMConfigKeys.OZONE_OM_GRPC_PORT_KEY,
// TODO HDDS-2856
+ OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_HTTPS_ADDRESS_KEY,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD
));
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index fdbf17d9a4,e38c413e74..69977648a1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@@ -102,9 -108,8 +109,9 @@@ public class TestOzoneShellHA
private static String testFilePathString;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
- private static OzoneShell ozoneShell = null;
- private static OzoneAdmin ozoneAdminShell = null;
- private static S3Shell s3Shell = null;
+ private OzoneShell ozoneShell = null;
+ private OzoneAdmin ozoneAdminShell = null;
++ private S3Shell s3Shell = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final ByteArrayOutputStream err = new ByteArrayOutputStream();
@@@ -170,6 -173,8 +175,9 @@@
@Before
public void setup() throws UnsupportedEncodingException {
+ ozoneShell = new OzoneShell();
+ ozoneAdminShell = new OzoneAdmin();
++ s3Shell = new S3Shell();
System.setOut(new PrintStream(out, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(err, false, DEFAULT_ENCODING));
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index deb77db639,d91d8519a9..c6a6745d5f
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -411,21 -356,7 +411,22 @@@ enum Status
LAYOUT_FEATURE_FINALIZATION_FAILED = 72;
PREPARE_FAILED = 73;
NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74;
+ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION = 75;
+
- TENANT_NOT_FOUND = 75;
- TENANT_ALREADY_EXISTS = 76;
- INVALID_TENANT_ID = 77;
++ TENANT_NOT_FOUND = 76;
++ TENANT_ALREADY_EXISTS = 77;
++ INVALID_TENANT_ID = 78;
+
- ACCESS_ID_NOT_FOUND = 78;
- TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 79;
- INVALID_TENANT_USERNAME = 80;
- INVALID_ACCESS_ID = 81;
- TENANT_AUTHORIZER_ERROR = 82;
++ ACCESS_ID_NOT_FOUND = 79;
++ TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 80;
++ INVALID_TENANT_USERNAME = 81;
++ INVALID_ACCESS_ID = 82;
++ TENANT_AUTHORIZER_ERROR = 83;
+
- VOLUME_IS_REFERENCED = 83;
- TENANT_NOT_EMPTY = 84;
++ VOLUME_IS_REFERENCED = 84;
++ TENANT_NOT_EMPTY = 85;
+
- FEATURE_NOT_ENABLED = 85;
++ FEATURE_NOT_ENABLED = 86;
}
/**
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d2848748a1,4e051875d7..0e27ac0565
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -67,10 -69,8 +69,11 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneManagerVersion;
+ import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
@@@ -127,12 -124,9 +130,13 @@@ import org.apache.hadoop.ozone.om.helpe
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.TenantStateList;
+import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
+import org.apache.hadoop.ozone.om.helpers.TenantUserList;
+ import org.apache.hadoop.ozone.om.lock.OzoneLockProvider;
import org.apache.hadoop.ozone.om.protocol.OMInterServiceProtocol;
import org.apache.hadoop.ozone.om.protocol.OMConfiguration;
import org.apache.hadoop.ozone.om.protocolPB.OMInterServiceProtocolClientSideImpl;
@@@ -300,6 -301,7 +314,7 @@@ public final class OzoneManager extend
private final Text omRpcAddressTxt;
private OzoneConfiguration configuration;
private RPC.Server omRpcServer;
- private GrpcOzoneManagerServer omS3gGrpcServer;
++ private GrpcOzoneManagerServer omS3gGrpcServer;
private InetSocketAddress omRpcAddress;
private String omId;
@@@ -339,7 -340,9 +354,9 @@@
private JvmPauseMonitor jvmPauseMonitor;
private final SecurityConfig secConfig;
private S3SecretManager s3SecretManager;
+ private final boolean isOmGrpcServerEnabled;
private volatile boolean isOmRpcServerRunning = false;
- private volatile boolean isOmGrpcServerRunning = false;
++ private volatile boolean isOmGrpcServerRunning = false;
private String omComponent;
private OzoneManagerProtocolServerSideTranslatorPB omServerProtocol;
@@@ -1964,6 -1991,6 +2046,9 @@@
if (httpServer != null) {
httpServer.stop();
}
++ if (multiTenantManager != null) {
++ multiTenantManager.stop();
++ }
stopTrashEmptier();
metadataManager.stop();
metrics.unRegister();
@@@ -3910,21 -3741,20 +4050,21 @@@
throws IOException {
Pair<String, String> resolved;
- try {
- if (isAclEnabled) {
- InetAddress remoteIp = Server.getRemoteIp();
- resolved = resolveBucketLink(requested, new HashSet<>(),
- getRemoteUser(),
- remoteIp,
- remoteIp != null ? remoteIp.getHostName() :
- omRpcAddress.getHostName());
- } else {
- resolved = resolveBucketLink(requested, new HashSet<>(),
- null, null, null);
+ if (isAclEnabled) {
- UserGroupInformation ugi = Server.getRemoteUser();
++ UserGroupInformation ugi = getRemoteUser();
+ if (getS3Auth() != null) {
+ ugi = UserGroupInformation.createRemoteUser(
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId()));
}
- } catch (Throwable t) {
- throw t;
+ InetAddress remoteIp = Server.getRemoteIp();
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ ugi,
+ remoteIp,
+ remoteIp != null ? remoteIp.getHostName() :
+ omRpcAddress.getHostName());
+ } else {
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ null, null, null);
}
return new ResolvedBucket(requested, resolved);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 60a888e446,7b1f4185b6..4302327cf3
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@@ -31,8 -31,13 +31,14 @@@ public enum OMLayoutFeature implements
////////////////////////////// //////////////////////////////
INITIAL_VERSION(0, "Initial Layout Version"),
- // TODO: Make this 2 after bringing in EC
- MULTITENANCY_SCHEMA(1, "Multi-Tenancy Schema");
+ ERASURE_CODED_STORAGE_SUPPORT(1, "Ozone version with built in support for"
+ + " Erasure Coded block data storage."),
+
+ BUCKET_LAYOUT_SUPPORT(2, "Ozone version supporting bucket " +
+ "layouts and introducing the FILE_SYSTEM_OPTIMIZED and OBJECT_STORE " +
- "bucket layout types.");
++ "bucket layout types."),
+
++ MULTITENANCY_SCHEMA(3, "Multi-Tenancy Schema");
/////////////////////////////// /////////////////////////////
// Example OM Layout Feature with Actions
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 4dffc39e22,ecec94592d..94b9b6d56c
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -48,8 -48,11 +51,12 @@@ import org.apache.hadoop.ozone.om.ratis
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+ import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+ import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
@@@ -78,15 -80,12 +86,17 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index 891c48ca96,e9ddc08815..ca8d32b48d
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@@ -71,9 -66,7 +66,7 @@@ public class OzoneClientProducer
@Produces
public synchronized OzoneClient createClient() throws WebApplicationException,
IOException {
- if (client == null) {
- client = createOzoneClient();
- }
- client = getClient(ozoneConfiguration);
++ client = getClient(ozoneConfiguration);
return client;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org