You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2021/06/02 13:06:38 UTC
[ozone] 01/03: Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 9c4008d5404d310ddc4e7cf5be22b80156692ebf
Merge: 132911c 0b4779c
Author: Elek Márton <el...@apache.org>
AuthorDate: Tue May 25 13:30:16 2021 +0200
Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
.github/buildenv/Dockerfile | 20 --
.github/buildenv/entrypoint.sh | 18 -
.github/workflows/post-commit.yml | 117 +++++--
.gitignore | 3 +
CONTRIBUTING.md | 10 +-
NOTICE.txt | 2 +-
README.md | 2 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 3 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +-
hadoop-hdds/common/pom.xml | 5 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 95 +++--
.../hadoop/hdds/client/RatisReplicationConfig.java | 15 +
.../hadoop/hdds/client/ReplicationConfig.java | 55 +++
.../hdds/client/StandaloneReplicationConfig.java | 9 +
.../hdds/ratis/ServerNotLeaderException.java | 15 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../hdds/scm/container/ContainerException.java | 14 +-
.../scm/container/ContainerNotFoundException.java | 4 +-
.../ContainerReplicaNotFoundException.java | 4 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 7 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 58 ++-
.../scm/pipeline/PipelineNotFoundException.java | 8 +-
.../pipeline/UnknownPipelineStateException.java | 8 +-
.../protocol/StorageContainerLocationProtocol.java | 18 +
.../security/token/ContainerTokenIdentifier.java | 103 ++++++
.../security/token/OzoneBlockTokenIdentifier.java | 146 +++-----
.../security/token/ShortLivedTokenIdentifier.java | 112 ++++++
.../hadoop/hdds/security/x509/SecurityConfig.java | 16 +
.../hadoop/hdds/security/x509/crl}/CRLCodec.java | 2 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 1 -
.../hdds/security/x509/crl}/CRLInfoCodec.java | 3 +-
.../hdds/security/x509/crl/package-info.java | 24 ++
.../org/apache/hadoop/hdds/utils/db/Codec.java | 0
.../apache/hadoop/hdds/utils/db/package-info.java | 22 ++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 10 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 +
.../common/src/main/resources/ozone-default.xml | 164 ++++++---
.../hadoop/hdds/client/TestReplicationConfig.java | 147 ++++++++
.../hdds/ratis/TestServerNotLeaderException.java | 82 +++++
.../ozone/container/ContainerTestHelper.java | 146 ++++----
hadoop-hdds/container-service/pom.xml | 12 +
.../hdds/datanode/metadata/CRLDBDefinition.java | 99 ++++++
.../hdds/datanode/metadata/DatanodeCRLStore.java | 84 +++++
.../datanode/metadata/DatanodeCRLStoreImpl.java | 127 +++++++
.../hdds/datanode/metadata/package-info.java | 22 ++
.../apache/hadoop/ozone/HddsDatanodeService.java | 19 +-
.../container/common/impl/HddsDispatcher.java | 25 +-
.../common/statemachine/DatanodeConfiguration.java | 30 ++
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../container/common/utils/HddsVolumeUtil.java | 11 +
.../ozone/container/common/volume/HddsVolume.java | 13 +
.../container/common/volume/MutableVolumeSet.java | 28 +-
.../container/keyvalue/KeyValueContainer.java | 6 +
.../container/keyvalue/KeyValueContainerData.java | 2 +
.../ozone/container/keyvalue/KeyValueHandler.java | 3 +-
.../container/keyvalue/helpers/BlockUtils.java | 4 +-
.../container/keyvalue/helpers/ChunkUtils.java | 42 ++-
.../keyvalue/impl/ChunkManagerDispatcher.java | 10 +-
.../keyvalue/impl/ChunkManagerFactory.java | 5 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 32 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 15 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 21 +-
.../stream/DirectoryServerDestination.java | 35 +-
.../container/stream/DirectoryServerSource.java | 57 +++
.../container/stream/DirstreamClientHandler.java | 143 ++++++++
.../container/stream/DirstreamServerHandler.java | 135 +++++++
.../ozone/container/stream/StreamingClient.java | 101 ++++++
.../container/stream/StreamingDestination.java | 31 +-
.../ozone/container/stream/StreamingServer.java | 94 +++++
.../ozone/container/stream/StreamingSource.java | 35 +-
.../ozone/container/stream/package-info.java | 22 ++
.../metadata/TestDatanodeCRLStoreImpl.java | 127 +++++++
.../hdds/datanode/metadata/package-info.java} | 31 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 6 +-
.../container/common/TestBlockDeletingService.java | 47 ++-
.../TestSchemaOneBackwardsCompatibility.java | 18 +
.../impl/TestContainerDeletionChoosingPolicy.java | 31 +-
.../common/impl/TestContainerPersistence.java | 3 +-
.../statemachine/TestDatanodeConfiguration.java | 15 +
.../container/keyvalue/ChunkLayoutTestInfo.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 14 +-
.../keyvalue/impl/TestFilePerChunkStrategy.java | 3 +-
.../container/stream/TestStreamingServer.java | 127 +++++++
hadoop-hdds/docs/content/interface/ReconApi.md | 390 +++++++++------------
hadoop-hdds/docs/content/interface/ReconApi.zh.md | 356 ++++++++-----------
hadoop-hdds/docs/content/interface/S3.md | 21 ++
hadoop-hdds/framework/pom.xml | 6 +
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 11 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 14 +
.../hadoop/hdds/scm/protocolPB/OzonePBHelper.java | 82 +++++
...inerLocationProtocolClientSideTranslatorPB.java | 64 +++-
.../SCMBlockLocationFailoverProxyProvider.java | 136 +++----
.../SCMContainerLocationFailoverProxyProvider.java | 158 +++++----
.../SCMSecurityProtocolFailoverProxyProvider.java | 101 +++---
.../hadoop/hdds}/security/OzoneSecretKey.java | 4 +-
.../hadoop/hdds}/security/OzoneSecretManager.java | 2 +-
.../hdds}/security/OzoneSecurityException.java | 2 +-
.../apache/hadoop/hdds/security/package-info.java | 31 +-
.../hdds/security/token/BlockTokenVerifier.java | 168 +++------
.../security/token/CompositeTokenVerifier.java | 47 +++
.../token/ContainerTokenSecretManager.java | 48 +++
.../security/token/ContainerTokenVerifier.java | 51 +++
.../hdds/security/token/NoopTokenVerifier.java | 38 +-
.../token/ShortLivedTokenSecretManager.java | 107 ++++++
.../security/token/ShortLivedTokenVerifier.java | 135 +++++++
.../hadoop/hdds/security/token/TokenVerifier.java | 64 +++-
.../certificate/authority/CertificateServer.java | 4 +-
.../certificate/authority/DefaultCAServer.java | 3 +-
.../certificate/authority/DefaultCRLApprover.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 15 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 2 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 30 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 155 ++++----
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 4 +-
.../hdds/scm/protocol/TestReplicationConfig.java | 86 -----
.../security/token/TestBlockTokenVerifier.java | 75 ++++
.../security/token/TestContainerTokenVerifier.java | 75 ++++
.../token/TestOzoneBlockTokenIdentifier.java | 2 +-
.../hdds/security/token/TokenVerifierTests.java | 242 +++++++++++++
.../certificate/authority/TestDefaultCAServer.java | 6 +-
.../x509/certificate/utils/TestCRLCodec.java | 1 +
.../src/main/proto/ScmAdminProtocol.proto | 31 ++
.../interface-client/src/main/proto/hdds.proto | 16 +
.../src/main/proto/ScmServerProtocol.proto | 5 +
.../src/main/proto/ScmServerSecurityProtocol.proto | 28 ++
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 16 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 51 ++-
.../hdds/scm/container/ContainerReportHandler.java | 53 +--
.../scm/container/ContainerStateManagerImpl.java | 11 +-
.../IncrementalContainerReportHandler.java | 54 +--
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 30 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 22 ++
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 16 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 29 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 192 +++++-----
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 35 +-
.../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 5 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 19 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 94 +++--
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 1 +
.../hdds/scm/node/NodeDecommissionManager.java | 31 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 16 +
...lockLocationProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 49 ++-
.../hdds/scm/server/SCMClientProtocolServer.java | 92 +++--
.../hdds/scm/server/SCMDBCheckpointServlet.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 33 +-
.../hdds/scm/server/StorageContainerManager.java | 143 ++++++--
.../hadoop/hdds/scm/container/MockNodeManager.java | 21 +-
.../scm/container/TestContainerReportHandler.java | 4 +-
.../TestIncrementalContainerReportHandler.java | 96 ++++-
.../algorithms/TestContainerPlacementFactory.java | 51 ++-
.../TestSCMContainerPlacementCapacity.java | 37 +-
.../TestSCMContainerPlacementRackAware.java | 105 ++++--
.../TestSCMContainerPlacementRandom.java | 32 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 76 ++++
.../hdds/scm/node/TestNodeDecommissionManager.java | 33 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 4 +-
.../hdds/scm/cli/ContainerOperationClient.java | 30 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 4 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 121 ++++---
.../hadoop/ozone/client/OzoneKeyLocation.java | 15 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 57 ++-
.../client/io/BlockOutputStreamEntryPool.java | 2 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 5 +-
.../ozone/client/protocol/ClientProtocol.java | 65 ++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 119 +++++--
.../hadoop/ozone/client/MockBlockAllocator.java} | 36 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 73 ++++
.../hadoop/ozone/client/MockOmTransport.java | 276 +++++++++++++++
.../ozone/client/MockXceiverClientFactory.java | 72 ++++
.../hadoop/ozone/client/MockXceiverClientSpi.java | 180 ++++++++++
.../ozone/client/SinglePipelineBlockAllocator.java | 90 +++++
.../hadoop/ozone/client/TestOzoneClient.java | 174 +++++++++
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 39 +--
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 11 +
.../ozone/security/acl/IAccessAuthorizer.java | 1 -
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/dependency.sh | 50 +++
hadoop-ozone/dev-support/checks/junit.sh | 2 +-
.../src/main/compose/ozone-ha/docker-compose.yaml | 10 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 16 +-
.../src/main/compose/ozonesecure-ha/docker-config | 3 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 5 +
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 235 +++++++++++++
.../test.sh => license/update-jar-report.sh} | 28 +-
.../src/main/smoketest/admincli/container.robot | 14 +-
.../src/main/smoketest/admincli/datanode.robot | 6 +-
.../src/main/smoketest/admincli/pipeline.robot | 7 +-
.../smoketest/admincli/replicationmanager.robot | 7 +-
.../src/main/smoketest/admincli/safemode.robot | 6 +-
.../dist/src/main/smoketest/admincli/scmha.robot | 2 +-
.../dist/src/main/smoketest/cli/envvars.robot | 15 +
.../dist/src/main/smoketest/commonlib.robot | 11 +
.../dist/src/main/smoketest/freon/remove.robot | 43 +++
hadoop-ozone/dist/src/shell/ozone/ozone | 7 +-
.../dist/src/shell/ozone/ozone-functions.sh | 3 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 9 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 12 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 33 ++
.../hadoop/ozone/TestContainerOperations.java | 21 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 78 ++++-
.../hadoop/ozone/TestStorageContainerManager.java | 32 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 3 +-
.../TestContainerStateMachineFailureOnRead.java | 6 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 24 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 12 -
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 11 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 151 ++++----
.../server/TestSecureContainerServer.java | 234 ++++++++-----
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 243 +++++++++++++
.../TestDatanodeHddsVolumeFailureDetection.java | 289 +++++++++++++++
.../ozone/freon/TestFreonWithDatanodeRestart.java | 126 -------
.../ozone/freon/TestHadoopDirTreeGenerator.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 5 +-
.../ozone/freon/TestOzoneClientKeyGenerator.java | 94 -----
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 19 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 274 +++++++++------
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 73 ++--
.../src/main/proto/OmClientProtocol.proto | 8 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 27 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 95 +++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../om/request/bucket/OMBucketDeleteRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 9 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../om/request/s3/security/S3GetSecretRequest.java | 14 +-
.../request/s3/security/S3RevokeSecretRequest.java | 155 ++++++++
.../s3/security/S3RevokeSecretResponse.java | 57 +++
...OzoneManagerProtocolServerSideTranslatorPB.java | 27 +-
.../security/OzoneBlockTokenSecretManager.java | 86 +----
.../OzoneDelegationTokenSecretManager.java | 1 +
.../security/TestOzoneBlockTokenSecretManager.java | 285 ++++++++-------
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 32 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 32 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 -
hadoop-ozone/ozonefs-hadoop3/pom.xml | 10 -
hadoop-ozone/ozonefs/pom.xml | 10 -
hadoop-ozone/recon/pom.xml | 2 +-
.../ozone/recon/scm/ReconContainerManager.java | 107 ++++--
.../recon/scm/ReconContainerReportHandler.java | 24 +-
.../ReconIncrementalContainerReportHandler.java | 2 +-
.../recon/spi/StorageContainerServiceProvider.java | 8 +
.../impl/StorageContainerServiceProviderImpl.java | 6 +
.../ozone/recon/api/TestContainerEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 8 +
.../scm/AbstractReconContainerManagerTest.java | 30 ++
.../ozone/recon/scm/TestReconContainerManager.java | 50 ++-
...TestReconIncrementalContainerReportHandler.java | 3 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 17 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 283 +++++++++++++--
.../s3/endpoint/ListMultipartUploadsResult.java | 53 +--
.../endpoint/PutBucketAclRequestUnmarshaller.java | 85 +++++
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 342 ++++++++++++++++++
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 235 +++++++++++++
.../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 72 ++++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +
.../hadoop/ozone/client/OzoneBucketStub.java | 28 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 24 ++
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 266 ++++++++++++++
.../{TestBucketGet.java => TestBucketList.java} | 76 ++--
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 6 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 56 ++-
.../src/test/resources/groupAccessControlList.xml | 57 +++
.../src/test/resources/userAccessControlList.xml | 42 +++
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 10 +
.../ContainerCommands.java} | 130 +++----
.../ozone/debug/container/ExportSubcommand.java | 75 ++++
.../ozone/debug/container/InfoSubcommand.java | 53 +++
.../container/ListSubcommand.java} | 53 ++-
.../hadoop/ozone/debug/container/package-info.java | 22 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 7 +
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 2 +-
...OmBucketGenerator.java => OmBucketRemover.java} | 36 +-
...etGenerator.java => OzoneClientKeyRemover.java} | 60 ++--
.../hadoop/ozone/freon/RandomKeyGenerator.java | 137 ++++++++
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/StreamingGenerator.java | 144 ++++++++
.../containergenerator/GeneratorDatanode.java | 2 +-
.../GenerateOzoneRequiredConfigurations.java | 44 ++-
.../ozone/genesis/BenchmarkChunkManager.java | 4 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 28 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 27 +-
.../hadoop/ozone/shell/s3/GetS3SecretHandler.java | 11 +-
...cretHandler.java => RevokeS3SecretHandler.java} | 57 +--
.../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 3 +-
.../TestGenerateOzoneRequiredConfigurations.java | 52 ++-
pom.xml | 59 +---
315 files changed, 11476 insertions(+), 3462 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org