You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/04/25 19:32:25 UTC
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2ea9e78bf1bd041a140a18c5bf7c0110a7fcc570
Merge: 55d2e06cf2 de9fcca1c3
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Mon Apr 25 18:39:39 2022 +0200
Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
.github/workflows/post-commit.yml | 50 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../hdds/scm/storage/BlockExtendedInputStream.java | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 177 ++++
.../hdds/scm/storage/ExtendedInputStream.java | 91 ++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 134 +++
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 16 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 29 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../common/src/main/resources/ozone-default.xml | 117 ++-
.../hdds/client/TestECReplicationConfig.java | 91 ++
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +++-
.../hdds/scm/container/TestContainerInfo.java | 86 ++
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/common/helpers/TestExcludeList.java | 67 ++
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 ++
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 54 +-
.../common/src/test/resources/auditlog.properties | 2 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++++
hadoop-hdds/docs/content/feature/OM-HA.md | 37 +
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 ++
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 ++
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +++
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 ++
.../rawcoder/AbstractNativeRawEncoder.java | 99 ++
.../rawcoder/ByteArrayDecodingState.java | 132 +++
.../rawcoder/ByteArrayEncodingState.java | 105 ++
.../rawcoder/ByteBufferDecodingState.java | 143 +++
.../rawcoder/ByteBufferEncodingState.java | 110 ++
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 ++
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 ++
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 ++
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 ++
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 ++
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 ++
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 ++
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 ++
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 ++
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 ++
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 ++++++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++++++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 ++
.../ozone/erasurecode/TestCodecRegistry.java | 159 +++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++++++++
.../apache/ozone/erasurecode}/package-info.java | 12 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 ++
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 ++
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +++
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 ++
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 1 +
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 20 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ReplicationManager.java | 20 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 ++++++
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 10 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +++
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 32 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 30 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 ++++-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 96 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../scm/container/TestContainerManagerImpl.java | 10 +
.../scm/container/TestContainerReportHandler.java | 181 +++-
.../hdds/scm/container/TestReplicationManager.java | 72 +-
.../algorithms/TestContainerPlacementFactory.java | 8 +
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +++---
.../hdds/scm/ha/TestReplicationAnnotation.java | 5 +
.../hdds/scm/pipeline/MockPipelineManager.java | 54 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 +++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 49 +
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 61 ++
.../pipeline/TestWritableECContainerProvider.java | 446 ++++++++
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 37 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 75 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 192 ++++
hadoop-ozone/client/pom.xml | 4 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 ++
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 420 ++++++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 ++
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 ++
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 +++++
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 +++++++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 +++++
.../io/ECBlockReconstructedStripeInputStream.java | 631 ++++++++++++
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 706 +++++++++++++
.../client/io/InsufficientLocationsException.java | 26 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 233 ++++-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 ++-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 +++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1066 ++++++++++++++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 28 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 ++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 9 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 58 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 ++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 41 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 ++
.../dist/src/main/smoketest/ec/basic.robot | 64 ++
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 8 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 2 +-
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
...g4j2.properties => s3g-audit-log4j2.properties} | 8 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 70 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 235 +++--
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 435 +++++---
.../ozone/client/rpc/read/ECStreamTestUtil.java | 408 ++++++++
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 111 ++
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../client/rpc/read/TestECBlockInputStream.java | 522 ++++++++++
.../rpc/read/TestECBlockInputStreamProxy.java | 399 ++++++++
.../read/TestECBlockReconstructedInputStream.java | 379 +++++++
.../TestECBlockReconstructedStripeInputStream.java | 724 +++++++++++++
.../ozone/client/rpc/read/TestInputStreamBase.java | 23 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../client/rpc/read/TestKeyInputStreamEC.java | 118 +++
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 ++--
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 3 +
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 -
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 156 ++-
.../src/main/proto/OmClientProtocol.proto | 20 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 66 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 ++
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 ++
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/request/bucket/OMBucketCreateRequest.java | 28 +
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 63 +-
.../request/file/OMFileCreateRequestWithFSO.java | 15 +-
.../om/request/key/OMAllocateBlockRequest.java | 37 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 31 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 58 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 30 +-
.../S3MultipartUploadCompleteRequest.java | 41 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 5 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 11 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 ++++-
.../hadoop/ozone/om/TestKeyDeletingService.java | 7 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 ++
.../apache/hadoop/ozone/om/TestTrashService.java | 2 +
.../ozone/om/lock/TestOzoneLockProvider.java | 108 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../request/validation/TestRequestValidations.java | 29 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +++
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +++-
.../org.apache.hadoop.security.token.DtFetcher} | 2 +-
...g.apache.hadoop.security.token.TokenIdentifier} | 6 +-
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 5 +
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../scm/ReconStorageContainerManagerFacade.java | 7 +
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
hadoop-ozone/s3gateway/pom.xml | 4 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 ++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 190 ++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 75 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 183 ++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 53 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 52 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 75 ++
.../hadoop/ozone/client/OzoneBucketStub.java | 69 +-
...ewayMetrics.java => TestS3GatewayAuditLog.java} | 102 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 5 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 173 +++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 +++
.../src/test/resources/auditlog.properties | 4 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 109 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 6 +
411 files changed, 26428 insertions(+), 2048 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index cb3b3e6fd9,436d5ab39a..6c76ca6156
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -3039,17 -3096,10 +3095,17 @@@
OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds.
</description>
</property>
-
+ <property>
+ <name>ozone.om.grpc.maximum.response.length</name>
+ <value>134217728</value>
+ <tag>OZONE, OM, S3GATEWAY</tag>
+ <description>
+ OM/S3GATEWAY OMRequest, OMResponse over grpc max message length (bytes).
+ </description>
+ </property>
<property>
<name>ozone.default.bucket.layout</name>
- <value>OBJECT_STORE</value>
+ <value/>
<tag>OZONE, MANAGEMENT</tag>
<description>
Default bucket layout used by Ozone Manager during bucket creation when a client does not specify the
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index e28907771d,c8af3e6e4b..bc1ed8d6f5
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -317,9 -320,17 +331,21 @@@ public final class OMConfigKeys
"ozone.om.admin.protocol.wait.between.retries";
public static final long OZONE_OM_ADMIN_PROTOCOL_WAIT_BETWEEN_RETRIES_DEFAULT
= 1000;
-
+ public static final String OZONE_OM_TRANSPORT_CLASS =
+ "ozone.om.transport.class";
+ public static final String OZONE_OM_TRANSPORT_CLASS_DEFAULT =
+ "org.apache.hadoop.ozone.om.protocolPB"
+ + ".Hadoop3OmTransportFactory";
+ public static final String OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT =
+ "ozone.om.unflushed.transaction.max.count";
+ public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
+ = 10000;
+
+ /**
+ * This configuration shall be enabled to utilize the functionality of the
+ * fine-grained KEY_PATH_LOCK.
+ */
+ public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED =
+ "ozone.om.key.path.lock.enabled";
+ public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false;
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
index 37f6b84f3d,001322e2af..985b3e76ba
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
@@@ -133,38 -133,36 +133,34 @@@ public class OMFailoverProxyProvider<T
this.omProxyInfos = new HashMap<>();
this.omNodeIDList = new ArrayList<>();
- Collection<String> omServiceIds = Collections.singletonList(omSvcId);
+ Collection<String> omNodeIds = OmUtils.getActiveOMNodeIds(config,
+ omSvcId);
- for (String serviceId : OmUtils.emptyAsSingletonNull(omServiceIds)) {
- Collection<String> omNodeIds = OmUtils.getActiveOMNodeIds(config,
- serviceId);
+ for (String nodeId : OmUtils.emptyAsSingletonNull(omNodeIds)) {
- for (String nodeId : OmUtils.emptyAsSingletonNull(omNodeIds)) {
+ String rpcAddrKey = ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
+ omSvcId, nodeId);
+ String rpcAddrStr = OmUtils.getOmRpcAddress(config, rpcAddrKey);
+ if (rpcAddrStr == null) {
+ continue;
+ }
- String rpcAddrKey = ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
- serviceId, nodeId);
- String rpcAddrStr = OmUtils.getOmRpcAddress(config, rpcAddrKey);
- if (rpcAddrStr == null) {
- continue;
- }
+ OMProxyInfo omProxyInfo = new OMProxyInfo(omSvcId, nodeId,
+ rpcAddrStr);
- OMProxyInfo omProxyInfo = new OMProxyInfo(serviceId, nodeId,
- rpcAddrStr);
-
- if (omProxyInfo.getAddress() != null) {
- // For a non-HA OM setup, nodeId might be null. If so, we assign it
- // the default value
- if (nodeId == null) {
- nodeId = OzoneConsts.OM_DEFAULT_NODE_ID;
- }
- // ProxyInfo will be set during first time call to server.
- omProxies.put(nodeId, null);
- omProxyInfos.put(nodeId, omProxyInfo);
- omNodeIDList.add(nodeId);
- } else {
- LOG.error("Failed to create OM proxy for {} at address {}",
- nodeId, rpcAddrStr);
+ if (omProxyInfo.getAddress() != null) {
-
-
+ // For a non-HA OM setup, nodeId might be null. If so, we assign it
+ // the default value
+ if (nodeId == null) {
+ nodeId = OzoneConsts.OM_DEFAULT_NODE_ID;
}
+ // ProxyInfo will be set during first time call to server.
+ omProxies.put(nodeId, null);
+ omProxyInfos.put(nodeId, omProxyInfo);
+ omNodeIDList.add(nodeId);
+ } else {
+ LOG.error("Failed to create OM proxy for {} at address {}",
+ nodeId, rpcAddrStr);
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index fa33d81949,b23962169c..a40461dcb3
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -64,10 -66,13 +65,15 @@@ public class TestOzoneConfigurationFiel
xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms");
xmlPropsToSkipCompare.add("ozone.om.leader.election.minimum.timeout" +
".duration"); // Deprecated config
+ xmlPrefixToSkipCompare.add("ozone.s3g");
+ configurationPrefixToSkipCompare.add("ozone.s3g");
configurationPropsToSkipCompare
.add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION);
+ // Currently replication and type configs moved to server side.
+ configurationPropsToSkipCompare
+ .add(OzoneConfigKeys.OZONE_REPLICATION);
+ configurationPropsToSkipCompare
+ .add(OzoneConfigKeys.OZONE_REPLICATION_TYPE);
configurationPropsToSkipCompare
.add(OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY);
configurationPropsToSkipCompare
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 1e9e343034,9a0efc7d15..b08bfdb91f
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@@ -131,14 -137,9 +149,15 @@@ public abstract class EndpointBase impl
try {
client.getObjectStore().createS3Bucket(bucketName);
} catch (OMException ex) {
+ getMetrics().incCreateBucketFailure();
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
+ } else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
+ throw newError(S3ErrorTable.ACCESS_DENIED,
+ s3Auth.getAccessID(), ex);
+ } else if (ex.getResult() == ResultCodes.TIMEOUT ||
+ ex.getResult() == ResultCodes.INTERNAL_ERROR) {
+ throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
} else if (ex.getResult() != ResultCodes.BUCKET_ALREADY_EXISTS) {
// S3 does not return error for bucket already exists, it just
// returns the location.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org