You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2020/03/27 17:28:08 UTC
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-2665-ofs
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 3c5ec6a0346259fc82130729aabc726a5a3d17be
Merge: 5616dc3 7d132ce
Author: Siyao Meng <sm...@cloudera.com>
AuthorDate: Fri Mar 27 10:21:56 2020 -0700
Merge branch 'master' into HDDS-2665-ofs
.../comment-commands/debug.sh | 17 +-
.../comment-commands/help.sh | 25 +-
.../comment-commands/label.sh | 23 +-
.../comment-commands/pending.sh | 27 +-
.../comment-commands/ready.sh | 31 +-
.../comment-commands/retest.sh | 27 +-
.github/process-comment.sh | 56 +
.../workflows/comments.yaml | 27 +-
.github/workflows/post-commit.yml | 18 +-
.github/workflows/pr.yml | 18 +-
README.md | 4 +-
.../apache/hadoop/hdds/conf/RatisClientConfig.java | 6 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 9 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 63 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 108 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 11 +-
hadoop-hdds/common/pom.xml | 2 +-
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 141 +-
.../hadoop/hdds/annotation/InterfaceAudience.java | 73 +
.../hadoop/hdds/annotation/InterfaceStability.java | 69 +
.../hadoop/hdds/annotation}/package-info.java | 7 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java} | 26 +-
.../conf/DatanodeRatisServerConfig.java | 22 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 18 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 4 +-
.../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 4 +-
.../hadoop/hdds/fs/SpaceUsageCheckParams.java | 4 +-
.../hadoop/hdds/fs/SpaceUsagePersistence.java | 4 +-
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 4 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 6 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 122 +-
.../org/apache/hadoop/hdds/recon/ReconConfig.java | 93 ++
...erPlacementPolicy.java => PlacementPolicy.java} | 12 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 29 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 2 +-
.../common/helpers/StorageContainerException.java | 2 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 7 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 4 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 57 +-
...lockLocationProtocolClientSideTranslatorPB.java | 2 +-
.../scm/protocolPB/ScmBlockLocationProtocolPB.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 2 +-
.../StorageContainerLocationProtocolPB.java | 2 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 7 +-
.../security/token/OzoneBlockTokenIdentifier.java | 2 +-
.../security/token/OzoneBlockTokenSelector.java | 2 +-
.../x509/certificate/utils/package-info.java | 4 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 4 +-
.../org/apache/hadoop/hdds/utils/Scheduler.java | 5 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 41 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +-
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 4 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 1 +
.../common/InconsistentStorageStateException.java | 4 +-
.../ozone/common/OzoneChecksumException.java | 4 +-
.../org/apache/hadoop/ozone/common/Storage.java | 10 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 2 +-
.../helpers/ContainerCommandRequestPBHelper.java | 4 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 6 +-
.../hadoop/ozone/lock/ReadWriteLockable.java | 16 +-
.../src/main/proto/DatanodeContainerProtocol.proto | 1 +
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 182 ++-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 117 +-
.../apache/hadoop/hdds/ratis/TestRatisHelper.java | 53 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 8 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +-
.../apache/hadoop/ozone/lock/TestLockManager.java | 30 +
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../hadoop/ozone/HddsDatanodeHttpServer.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 48 +-
.../container/common/helpers/ContainerMetrics.java | 2 +-
.../container/common/impl/ChunkLayOutVersion.java | 70 +-
.../ozone/container/common/impl/ContainerData.java | 41 +-
.../container/common/impl/ContainerDataYaml.java | 5 +-
.../ozone/container/common/impl/ContainerSet.java | 14 +
.../container/common/impl/HddsDispatcher.java | 15 +-
.../ozone/container/common/interfaces/Handler.java | 18 +-
.../common/interfaces/VolumeChoosingPolicy.java | 2 +-
.../report/CommandStatusReportPublisher.java | 2 +-
.../common/report/ContainerReportPublisher.java | 2 +-
.../common/report/NodeReportPublisher.java | 2 +-
.../common/report/PipelineReportPublisher.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 28 +-
.../common/statemachine/EndpointStateMachine.java | 6 +-
.../common/statemachine/SCMConnectionManager.java | 66 +-
.../common/statemachine/StateContext.java | 60 +-
.../CreatePipelineCommandHandler.java | 3 +
.../commandhandler/DeleteBlocksCommandHandler.java | 2 +-
.../common/states/datanode/InitDatanodeState.java | 8 +-
.../states/datanode/RunningDatanodeState.java | 106 +-
.../states/endpoint/HeartbeatEndpointTask.java | 8 +-
.../states/endpoint/RegisterEndpointTask.java | 59 +-
.../states/endpoint/VersionEndpointTask.java | 4 +-
.../common/transport/server/ratis/CSMMetrics.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 69 +-
.../transport/server/ratis/DispatcherContext.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 106 +-
.../container/common/volume/AsyncChecker.java | 4 +-
.../ozone/container/common/volume/HddsVolume.java | 26 +-
.../container/common/volume/HddsVolumeChecker.java | 50 +-
.../common/volume/ImmutableVolumeSet.java | 49 +-
.../{VolumeSet.java => MutableVolumeSet.java} | 78 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../ozone/container/common/volume/VolumeSet.java | 518 +-----
.../container/keyvalue/KeyValueBlockIterator.java | 2 +-
.../container/keyvalue/KeyValueContainerData.java | 25 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 60 +-
.../container/keyvalue/helpers/ChunkUtils.java | 262 +--
.../keyvalue/impl/ChunkManagerDispatcher.java | 148 ++
.../keyvalue/impl/ChunkManagerDummyImpl.java | 115 +-
.../keyvalue/impl/ChunkManagerFactory.java | 45 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 297 ++++
...kManagerImpl.java => FilePerChunkStrategy.java} | 195 ++-
.../keyvalue/interfaces/ChunkManager.java | 15 +-
.../background/BlockDeletingService.java | 22 +-
.../container/ozoneimpl/ContainerController.java | 11 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 2 +-
.../ContainerMetadataScrubberMetrics.java | 2 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 33 +-
.../replication/DownloadAndImportReplicator.java | 8 +-
.../ozone/protocol/ReconDatanodeProtocol.java} | 29 +-
.../protocol/StorageContainerDatanodeProtocol.java | 2 +-
.../protocol/StorageContainerNodeProtocol.java | 2 +-
.../ozone/protocolPB/ReconDatanodeProtocolPB.java} | 24 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 1 +
.../proto/StorageContainerDatanodeProtocol.proto | 1 +
.../ozone/container/common/ContainerTestUtils.java | 59 +
.../ozone/container/common/SCMTestUtils.java | 2 +-
.../container/common/TestBlockDeletingService.java | 182 +--
.../container/common/TestChunkLayOutVersion.java | 21 +-
.../common/TestKeyValueContainerData.java | 2 +
.../container/common/helpers/TestBlockData.java | 6 +-
.../common/impl/TestContainerDataYaml.java | 10 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +
.../common/impl/TestContainerPersistence.java | 69 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 50 +-
.../container/common/interfaces/TestHandler.java | 3 +-
.../common/statemachine/TestStateContext.java | 9 +-
.../TestCloseContainerCommandHandler.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 6 +-
.../common/volume/TestHddsVolumeChecker.java | 101 +-
.../container/common/volume/TestVolumeSet.java | 12 +-
.../common/volume/TestVolumeSetDiskChecks.java | 11 +-
.../keyvalue/TestKeyValueBlockIterator.java | 8 +-
.../container/keyvalue/TestKeyValueContainer.java | 10 +-
.../keyvalue/TestKeyValueContainerCheck.java | 15 +-
.../TestKeyValueContainerMarkUnhealthy.java | 5 +-
.../container/keyvalue/TestKeyValueHandler.java | 7 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 4 +-
.../container/keyvalue/TestTarContainerPacker.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 48 +-
.../keyvalue/{ => impl}/TestBlockManagerImpl.java | 10 +-
.../TestChunkManager.java} | 51 +-
.../container/ozoneimpl/TestOzoneContainer.java | 11 +-
.../replication/TestReplicationSupervisor.java | 10 +-
.../docs/content/beyond/DockerCheatSheet.zh.md | 85 +
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 2 +-
.../docs/content/beyond/RunningWithHDFS.zh.md | 64 +
hadoop-hdds/docs/content/concept/Hdds.zh.md | 40 +
hadoop-hdds/docs/content/concept/Overview.zh.md | 50 +
.../docs/content/concept/OzoneManager.zh.md | 64 +
hadoop-hdds/docs/content/gdpr/GDPR in Ozone.zh.md | 36 +
hadoop-hdds/framework/pom.xml | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 2 +-
.../apache/hadoop/hdds/protocol}/package-info.java | 7 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 0
.../hdds/protocolPB/SCMSecurityProtocolPB.java | 0
.../hadoop/hdds/protocolPB/package-info.java | 0
.../hdds/security/token/BlockTokenException.java | 0
.../hdds/security/token/BlockTokenVerifier.java | 0
.../hadoop/hdds/security/token/TokenVerifier.java | 0
.../hadoop/hdds/security/token/package-info.java | 2 +-
.../x509/certificate/authority/BaseApprover.java | 0
.../certificate/authority/CertificateApprover.java | 0
.../certificate/authority/CertificateServer.java | 0
.../certificate/authority/CertificateStore.java | 0
.../certificate/authority/DefaultApprover.java | 0
.../certificate/authority/DefaultCAServer.java | 0
.../authority/PKIProfiles/DefaultCAProfile.java | 0
.../authority/PKIProfiles/DefaultProfile.java | 2 +-
.../authority/PKIProfiles/PKIProfile.java | 0
.../authority/PKIProfiles/package-info.java | 0
.../x509/certificate/authority/package-info.java | 0
.../x509/certificate/client/CertificateClient.java | 0
.../certificate/client/DNCertificateClient.java | 0
.../client/DefaultCertificateClient.java | 60 +-
.../certificate/client/OMCertificateClient.java | 0
.../x509/certificate/client/package-info.java | 0
.../security/x509/certificate/utils/CRLCodec.java | 0
.../x509/certificate/utils/package-info.java | 0
.../certificates/utils/CertificateSignRequest.java | 0
.../certificates/utils/SelfSignedCertificate.java | 0
.../x509/certificates/utils/package-info.java | 0
.../x509/exceptions/CertificateException.java | 0
.../security/x509/exceptions/package-info.java | 0
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 0
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 0
.../hdds/security/x509/keys/SecurityUtil.java | 0
.../hdds/security/x509/keys/package-info.java | 0
.../hadoop/hdds/security/x509/package-info.java | 99 ++
.../server/OzoneProtocolMessageDispatcher.java | 2 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 11 +-
.../hdds/server/http/AdminAuthorizedServlet.java} | 33 +-
.../hdds/server/{ => http}/BaseHttpServer.java | 173 +-
.../hadoop/hdds/server/http/FilterContainer.java | 32 +-
.../hadoop/hdds/server/http/FilterInitializer.java | 16 +-
.../hadoop/hdds/server/http/HtmlQuoting.java | 235 +++
.../apache/hadoop/hdds/server/http/HttpConfig.java | 75 +
.../hadoop/hdds/server/http/HttpRequestLog.java | 107 ++
.../hdds/server/http/HttpRequestLogAppender.java} | 45 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1706 ++++++++++++++++++++
.../hdds/server/{ => http}/LogStreamServlet.java | 2 +-
.../hadoop/hdds/server/http/NoCacheFilter.java | 55 +
.../hdds/server/{ => http}/ProfileServlet.java | 4 +-
.../server/{ => http}/PrometheusMetricsSink.java | 2 +-
.../hdds/server/{ => http}/PrometheusServlet.java | 2 +-
.../server/{ => http}/RatisDropwizardExports.java | 2 +-
.../{ => http}/RatisNameRewriteSampleBuilder.java | 2 +-
.../hdds/server/http/StaticUserWebFilter.java | 157 ++
.../hadoop/hdds/server/http/package-info.java | 10 +-
.../apache/hadoop/hdds/utils}/HddsServerUtil.java | 170 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 12 +-
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 0
.../hadoop/hdds/utils/MetaStoreIterator.java | 0
.../hadoop/hdds/utils/MetadataKeyFilters.java | 0
.../apache/hadoop/hdds/utils/MetadataStore.java | 2 +-
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 0
.../hadoop/hdds/utils}/ProtocolMessageMetrics.java | 2 +-
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 1 +
.../hadoop/hdds/utils/RocksDBStoreIterator.java | 0
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 0
.../hadoop/hdds/utils/db/BatchOperation.java | 0
.../hadoop/hdds/utils/db/ByteArrayCodec.java | 0
.../hadoop/hdds/utils/db/ByteArrayKeyValue.java | 0
.../org/apache/hadoop/hdds/utils/db/Codec.java | 0
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 0
.../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 0
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 0
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 5 +-
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 70 +-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 0
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 0
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 0
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 0
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 4 +-
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 28 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 0
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 0
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 28 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 4 +-
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 0
.../utils/db/SequenceNumberNotFoundException.java | 0
.../apache/hadoop/hdds/utils/db/StringCodec.java | 0
.../org/apache/hadoop/hdds/utils/db/Table.java | 19 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 0
.../apache/hadoop/hdds/utils/db/TableIterator.java | 0
.../apache/hadoop/hdds/utils/db/TypedTable.java | 24 +
.../hadoop/hdds/utils/db/cache/CacheKey.java | 0
.../hadoop/hdds/utils/db/cache/CacheResult.java | 0
.../hadoop/hdds/utils/db/cache/CacheValue.java | 0
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 0
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 4 +-
.../hadoop/hdds/utils/db/cache/package-info.java | 0
.../apache/hadoop/hdds/utils/db/package-info.java | 0
.../apache/hadoop/hdds}/utils/package-info.java | 10 +-
.../token/TestOzoneBlockTokenIdentifier.java | 0
.../hadoop/hdds/security/token/package-info.java | 0
.../x509/certificate/authority/MockApprover.java | 0
.../x509/certificate/authority/MockCAStore.java | 0
.../certificate/authority/TestDefaultCAServer.java | 0
.../certificate/authority/TestDefaultProfile.java | 0
.../x509/certificate/authority/package-info.java | 0
.../client/TestCertificateClientInit.java | 0
.../client/TestDefaultCertificateClient.java | 0
.../x509/certificate/utils/TestCRLCodec.java | 0
.../certificate/utils/TestCertificateCodec.java | 0
.../x509/certificate/utils/package-info.java | 0
.../certificates/TestCertificateSignRequest.java | 0
.../x509/certificates/TestRootCertificate.java | 0
.../security/x509/certificates/package-info.java | 0
.../security/x509/keys/TestHDDSKeyGenerator.java | 0
.../hdds/security/x509/keys/TestKeyCodec.java | 0
.../hdds/security/x509/keys/package-info.java | 0
.../hadoop/hdds/security/x509/package-info.java | 0
.../hdds/server/{ => http}/TestBaseHttpServer.java | 3 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 96 ++
.../hdds/server/http/TestHttpRequestLog.java | 51 +
.../server/http/TestHttpRequestLogAppender.java | 21 +-
.../hdds/server/{ => http}/TestProfileServlet.java | 6 +-
.../{ => http}/TestPrometheusMetricsSink.java | 2 +-
.../{ => http}/TestRatisDropwizardExports.java | 7 +-
.../server/{ => http}/TestRatisNameRewrite.java | 2 +-
.../hadoop/hdds/server/http/package-info.java | 10 +-
.../hadoop/hdds/utils/TestMetadataStore.java | 0
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 0
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 0
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 0
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 0
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 63 +
.../hdds/utils/db/TestTypedRDBTableStore.java | 18 +
.../hdds/utils/db/cache/TestTableCacheImpl.java | 0
.../hadoop/hdds/utils/db/cache/package-info.java | 0
.../apache/hadoop/hdds/utils/db/package-info.java | 0
.../apache/hadoop/hdds/utils/package-info.java} | 31 +-
.../src/test/resources/test.db.ini | 0
...onPolicy.java => SCMCommonPlacementPolicy.java} | 23 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 12 +-
.../hdds/scm/container/ReplicationManager.java | 27 +-
.../hdds/scm/container/SCMContainerManager.java | 39 +-
.../ContainerPlacementPolicyFactory.java | 22 +-
.../algorithms/SCMContainerPlacementCapacity.java | 4 +-
.../algorithms/SCMContainerPlacementRackAware.java | 12 +-
.../algorithms/SCMContainerPlacementRandom.java | 6 +-
.../scm/container/states/ContainerStateMap.java | 3 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 14 +
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 2 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 19 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 2 +-
.../hdds/scm/node/SCMNodeStorageStatMXBean.java | 2 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 4 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 11 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 2 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 13 +-
.../hdds/scm/pipeline/PipelineManagerMXBean.java | 2 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 428 +++++
.../hdds/scm/pipeline/PipelineReportHandler.java | 9 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 5 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 164 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 117 ++
.../hdds/scm/pipeline/SCMPipelineManager.java | 131 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 35 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 2 +-
...lockLocationProtocolServerSideTranslatorPB.java | 4 +-
...inerLocationProtocolServerSideTranslatorPB.java | 30 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 13 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 4 -
.../hadoop/hdds/scm/safemode/SafeModeHandler.java | 87 +-
.../hdds/scm/safemode/SafeModeNotification.java | 25 +
.../scm/server/OzoneStorageContainerManager.java | 12 +
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 27 +-
.../hdds/scm/server/SCMContainerMetrics.java | 2 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 87 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 2 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 8 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 6 +-
.../hdds/scm/server/StorageContainerManager.java | 63 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/server/StorageContainerManagerStarter.java | 9 +-
...ServerUtilTest.java => TestHddsServerUtil.java} | 114 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 1 +
.../scm/TestStorageContainerManagerHttpServer.java | 8 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 65 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 54 +-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../hdds/scm/container/TestReplicationManager.java | 7 +-
.../algorithms/TestContainerPlacementFactory.java | 7 +-
.../hdds/scm/node/TestContainerPlacement.java | 5 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 13 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +
.../scm/pipeline/MockRatisPipelineProvider.java | 43 +
.../TestPipelineDatanodesIntersection.java | 133 ++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 340 ++++
.../scm/pipeline/TestRatisPipelineProvider.java | 75 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 74 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +-
.../hdds/scm/safemode/TestSafeModeHandler.java | 11 +-
.../scm/server/TestSCMBlockProtocolServer.java | 2 +-
.../scm/server/TestSCMClientProtocolServer.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 16 +
hadoop-hdds/tools/pom.xml | 19 +
.../hdds/scm/cli}/ContainerOperationClient.java | 48 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 13 +-
.../DatanodeCommands.java} | 24 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 121 ++
.../hadoop/hdds/scm/cli/datanode/package-info.java | 9 +-
...bcommand.java => CreatePipelineSubcommand.java} | 65 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +-
.../hdds/scm/cli/pipeline/PipelineCommands.java | 1 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 8 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 52 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 7 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 2 +-
.../client/io/BlockOutputStreamEntryPool.java | 23 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 69 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 11 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 13 +-
.../ozone/OzoneIllegalArgumentException.java | 4 +-
.../hadoop/ozone/om/exceptions/OMException.java | 4 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 18 +-
.../org/apache/hadoop/ozone/om/ha/OMProxyInfo.java | 25 +-
.../ozone/om/helpers/EncryptionBucketInfo.java | 4 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 21 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 64 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 75 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 35 +-
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 22 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 59 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 16 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 123 ++
...OzoneManagerProtocolClientSideTranslatorPB.java | 63 +-
.../om/protocolPB/OzoneManagerProtocolPB.java | 2 +-
.../security/OzoneBlockTokenSecretManager.java | 4 +-
.../OzoneDelegationTokenSecretManager.java | 15 +-
.../security/OzoneDelegationTokenSelector.java | 2 +-
.../hadoop/ozone/security/OzoneSecretKey.java | 4 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 4 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 2 +-
.../ozone/security/OzoneTokenIdentifier.java | 4 +-
.../ozone/security/acl/IAccessAuthorizer.java | 4 +-
.../hadoop/ozone/security/acl/OzoneAclConfig.java | 3 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 2 +-
.../src/main/proto/OzoneManagerProtocol.proto | 7 +
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +-
hadoop-ozone/csi/pom.xml | 2 +-
.../dev-support/checks/_mvn_unit_report.sh | 36 +-
hadoop-ozone/dev-support/checks/integration.sh | 7 +-
hadoop-ozone/dev-support/checks/unit.sh | 12 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 18 +-
.../{Datanode.xml => Datanode1.xml} | 6 +-
.../{Datanode.xml => Datanode2.xml} | 6 +-
.../{Datanode.xml => Datanode3.xml} | 6 +-
.../dist/src/main/assemblies/ozone-src.xml | 4 +-
.../dist/src/main/compose/ozone-om-ha-s3/test.sh | 4 +
.../compose/ozone-topology/docker-compose.yaml | 28 +
.../{ozone/test.sh => ozone-topology/hdds-3084.sh} | 27 +-
.../dist/src/main/compose/ozone/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +
.../src/main/compose/ozonesecure-mr/docker-config | 2 +
.../src/main/compose/ozonesecure/docker-config | 8 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 48 +-
.../dist/src/main/smoketest/commonlib.robot | 1 +
.../dist/src/main/smoketest/freon/freon.robot | 36 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 2 +
.../dist/src/main/smoketest/recon/recon-api.robot | 45 +-
.../dist/src/main/smoketest/s3/webui.robot | 3 +-
.../scmcli/{pipeline.robot => datanode.robot} | 7 +-
.../dist/src/main/smoketest/scmcli/pipeline.robot | 7 +-
.../pipeline.robot => topology/loaddata.robot} | 10 +-
.../pipeline.robot => topology/readdata.robot} | 7 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 5 +
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 7 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 19 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 265 +--
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 33 +-
.../ozone/loadgenerators/AgedLoadGenerator.java | 98 ++
.../hadoop/ozone/loadgenerators/DataBuffer.java | 53 +
.../loadgenerators/FilesystemLoadGenerator.java | 69 +
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 101 ++
.../ozone/loadgenerators/LoadGenerator.java} | 24 +-
.../ozone/loadgenerators/RandomLoadGenerator.java | 68 +
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 23 +-
.../src/test/resources/log4j.properties | 10 +-
hadoop-ozone/insight/pom.xml | 6 -
.../hadoop/ozone/insight/BaseInsightPoint.java | 4 +-
...gSubcommandTest.java => TestLogSubcommand.java} | 14 +-
hadoop-ozone/integration-test/pom.xml | 113 --
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 214 ++-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 46 +
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 7 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 23 +
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 65 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 6 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 1 +
.../TestRatisPipelineCreateAndDestroy.java | 24 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 5 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 3 +
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 50 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 131 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 34 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 4 +-
.../hadoop/ozone/TestContainerOperations.java | 9 +-
.../TestContainerStateMachineIdempotency.java | 5 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 19 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 100 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 56 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 23 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 428 +----
.../rpc/TestCloseContainerHandlingByClient.java | 3 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 84 +-
.../rpc/TestContainerReplicationEndToEnd.java | 14 +-
.../client/rpc/TestContainerStateMachine.java | 11 +-
.../TestContainerStateMachineFailureOnRead.java | 224 +++
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 57 +-
.../client/rpc/TestFailureHandlingByClient.java | 31 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 5 +-
.../ozone/client/rpc/TestKeyInputStream.java | 3 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 31 +-
.../rpc/TestOzoneClientRetriesOnException.java | 74 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 14 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 3 +-
.../ozone/client/rpc/TestWatchForCommit.java | 5 +-
.../TestCloseContainerByPipeline.java | 7 +-
.../commandhandler/TestCloseContainerHandler.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 3 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 4 +-
.../container/ozoneimpl/TestRatisManager.java | 6 +-
.../container/server/TestContainerServer.java | 3 +-
.../server/TestSecureContainerServer.java | 3 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 85 +
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +-
.../hadoop/ozone/freon/TestDataValidate.java | 55 +-
.../freon/TestDataValidateWithDummyContainers.java | 18 +-
.../ozone/freon/TestFreonWithDatanodeRestart.java | 35 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 1 +
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 21 +
.../ozone/om/TestContainerReportWithKeys.java | 2 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 24 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 3 +
.../ozone/om/parser/TestOMRatisLogParser.java | 124 ++
.../hadoop/ozone/ozShell/TestOzoneShellHA.java | 143 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 200 +++
...stRecon.java => TestReconWithOzoneManager.java} | 89 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 142 ++
.../hadoop/ozone/scm/TestContainerSmallFile.java | 4 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 7 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 1 +
.../hadoop/ozone/scm/node/TestQueryNode.java | 5 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 138 ++
.../src/test/resources/ozone-site.xml | 5 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 171 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 2 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 8 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 45 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 182 ++-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 9 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 42 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 6 +-
.../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 3 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 14 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../metrics/OzoneManagerDoubleBufferMetrics.java | 49 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 46 +-
.../om/request/bucket/OMBucketCreateRequest.java | 17 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 10 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 9 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 13 +-
.../om/request/file/OMDirectoryCreateRequest.java | 271 +++-
.../ozone/om/request/file/OMFileCreateRequest.java | 150 +-
.../ozone/om/request/file/OMFileRequest.java | 153 +-
.../om/request/key/OMAllocateBlockRequest.java | 12 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 94 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 22 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 14 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 6 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 37 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 25 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 65 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 45 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 47 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 40 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 94 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 51 +-
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 48 +-
.../key/acl/prefix/OMPrefixSetAclRequest.java | 44 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 13 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 10 +-
.../S3InitiateMultipartUploadRequest.java | 74 +-
.../multipart/S3MultipartUploadAbortRequest.java | 102 +-
.../S3MultipartUploadCommitPartRequest.java | 100 +-
.../S3MultipartUploadCompleteRequest.java | 75 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../security/OMCancelDelegationTokenRequest.java | 7 +-
.../security/OMGetDelegationTokenRequest.java | 16 +-
.../security/OMRenewDelegationTokenRequest.java | 7 +-
.../om/request/volume/OMVolumeCreateRequest.java | 16 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 13 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 13 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 71 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 37 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 37 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 35 +-
.../response/file/OMDirectoryCreateResponse.java | 37 +-
.../om/response/file/OMFileCreateResponse.java | 12 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 45 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 29 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 7 +-
.../om/response/key/acl/OMKeyAclResponse.java | 29 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 46 +-
.../S3InitiateMultipartUploadResponse.java | 39 +-
.../multipart/S3MultipartUploadAbortResponse.java | 62 +-
.../S3MultipartUploadCommitPartResponse.java | 35 +-
.../S3MultipartUploadCompleteResponse.java | 64 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 28 +-
.../om/response/volume/OMVolumeCreateResponse.java | 2 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 43 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 1 +
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 33 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 4 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 33 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 31 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 33 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 3 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 33 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 31 +-
.../ozone/web/ozShell/s3/GetS3SecretHandler.java | 11 +-
.../ozShell/{bucket => s3}/S3BucketMapping.java | 8 +-
.../hadoop/ozone/web/ozShell/s3/S3Handler.java | 38 +-
.../hadoop/ozone/web/ozShell/s3/S3Shell.java | 1 -
.../web/ozShell/volume/AddAclVolumeHandler.java | 33 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 4 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 33 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 33 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 48 +
.../ozone/om/TestOzoneManagerHttpServer.java | 8 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 7 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 182 +--
.../ozone/om/request/TestOMRequestUtils.java | 3 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 4 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../om/request/file/TestOMFileCreateRequest.java | 40 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 92 ++
.../om/request/key/TestOMKeyCommitRequest.java | 42 +-
.../om/request/key/TestOMPrefixAclRequest.java | 101 ++
.../request/volume/TestOMVolumeCreateRequest.java | 20 +-
.../volume/acl/TestOMVolumeAddAclRequest.java | 33 +
.../volume/acl/TestOMVolumeRemoveAclRequest.java | 47 +-
.../volume/acl/TestOMVolumeSetAclRequest.java | 35 +
.../file/TestOMDirectoryCreateResponse.java | 37 +-
.../om/response/key/TestOMKeyCommitResponse.java | 11 +-
.../om/response/key/TestOMKeyCreateResponse.java | 4 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 6 +-
.../s3/multipart/TestS3MultipartResponse.java | 8 +-
hadoop-ozone/ozonefs-lib-legacy/pom.xml | 5 +-
.../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 92 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 48 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 12 +-
.../hadoop/fs/ozone/FilteredClassLoader.java | 1 +
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 6 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 4 +-
.../hadoop/fs/ozone/OzoneFSStorageStatistics.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +-
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 2 +-
.../org/apache/hadoop/fs/ozone/package-info.java | 4 +-
.../fs/ozone/TestOzoneFileSystemWithMocks.java | 120 +-
.../hadoop/fs/ozone/TestReadWriteStatistics.java | 4 +-
.../recon/codegen/ReconSchemaGenerationModule.java | 4 +-
...inition.java => ReconTaskSchemaDefinition.java} | 6 +-
.../recon/schema/UtilizationSchemaDefinition.java | 13 +
hadoop-ozone/recon/pom.xml | 29 +
.../hadoop/ozone/recon/ConfigurationProvider.java | 16 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 4 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 48 +-
.../recon/ReconGuiceServletContextListener.java | 2 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 6 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 7 +-
.../hadoop/ozone/recon/ReconSchemaManager.java | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 106 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 5 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 110 ++
...ainerKeyService.java => ContainerEndpoint.java} | 56 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 129 ++
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 107 ++
.../hadoop/ozone/recon/api/TaskStatusService.java | 7 +-
.../hadoop/ozone/recon/api/UtilizationService.java | 14 +-
.../hadoop/ozone/recon/api/package-info.java | 2 +-
.../recon/api/types/ClusterStateResponse.java | 198 +++
.../ozone/recon/api/types/ContainerKeyPrefix.java | 2 +-
.../ozone/recon/api/types/ContainerMetadata.java | 2 +-
.../ozone/recon/api/types/ContainersResponse.java | 13 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 88 +
.../ozone/recon/api/types/DatanodePipeline.java | 39 +-
.../types/DatanodeStorageReport.java} | 35 +-
.../ozone/recon/api/types/DatanodesResponse.java | 58 +
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 2 +-
.../hadoop/ozone/recon/api/types/KeysResponse.java | 13 +-
...Metadata.java => MissingContainerMetadata.java} | 46 +-
.../recon/api/types/MissingContainersResponse.java | 53 +
.../ozone/recon/api/types/PipelineMetadata.java | 218 +++
.../ozone/recon/api/types/PipelinesResponse.java | 59 +
.../hadoop/ozone/recon/api/types/package-info.java | 2 +-
.../ozone/recon/fsck/MissingContainerInfo.java | 68 +
.../ozone/recon/fsck/MissingContainerTask.java | 109 ++
.../recon/{persistence => fsck}/package-info.java | 4 +-
.../apache/hadoop/ozone/recon/package-info.java | 2 +-
.../recon/persistence/DataSourceConfiguration.java | 5 +-
.../persistence/DefaultDataSourceProvider.java | 5 +-
.../recon/persistence/JooqPersistenceModule.java | 3 +-
.../TransactionalMethodInterceptor.java | 2 +-
.../ozone/recon/persistence/package-info.java | 2 +-
.../recon/recovery/ReconOMMetadataManager.java | 2 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 5 +-
.../hadoop/ozone/recon/recovery/package-info.java | 2 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 73 +
.../ozone/recon/scm/ReconContainerManager.java | 55 +-
.../recon/scm/ReconContainerReportHandler.java | 74 +
.../recon/scm/ReconDatanodeProtocolServer.java | 31 +-
.../ReconIncrementalContainerReportHandler.java | 28 +-
.../ozone/recon/scm/ReconNewNodeHandler.java | 53 +
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 168 ++
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +-
.../ozone/recon/scm/ReconPipelineManager.java | 32 +-
.../recon/scm/ReconPipelineReportHandler.java | 2 +-
.../ozone/recon/scm/ReconPolicyProvider.java | 66 +
.../ozone/recon/scm/ReconSafeModeManager.java | 2 +-
.../hadoop/ozone/recon/scm/ReconScmTask.java | 106 ++
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 2 +-
.../scm/ReconStorageContainerManagerFacade.java | 71 +-
.../hadoop/ozone/recon/scm/package-info.java | 2 +-
.../recon/spi/ContainerDBServiceProvider.java | 13 +-
.../recon/spi/HddsDatanodeServiceProvider.java | 5 +-
.../recon/spi/OzoneManagerServiceProvider.java | 5 +-
.../recon/spi/StorageContainerServiceProvider.java | 5 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 15 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 100 +-
.../recon/spi/impl/ReconContainerDBProvider.java | 2 +-
.../impl/StorageContainerServiceProviderImpl.java | 2 +-
.../hadoop/ozone/recon/spi/impl/package-info.java | 2 +-
.../hadoop/ozone/recon/spi/package-info.java | 2 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 15 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 73 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 38 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 52 +-
.../ozone/recon/tasks/OMUpdateEventBatch.java | 2 +-
.../{ReconDBUpdateTask.java => ReconOmTask.java} | 4 +-
.../ozone/recon/tasks/ReconTaskController.java | 6 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 64 +-
.../hadoop/ozone/recon/tasks/package-info.java | 2 +-
.../webapps/recon/ozone-recon-web/LICENSE | 32 +
.../webapps/recon/ozone-recon-web/api/db.json | 315 ++--
.../webapps/recon/ozone-recon-web/package.json | 6 +-
.../webapps/recon/ozone-recon-web/src/App.less | 81 +
.../webapps/recon/ozone-recon-web/src/App.tsx | 2 +-
.../src/components/OverviewCard/OverviewCard.less | 13 +-
.../src/components/OverviewCard/OverviewCard.tsx | 17 +-
.../StorageBar.less} | 41 +-
.../src/components/StorageBar/StorageBar.tsx | 70 +
.../ozone-recon-web/src/makeRouteWithSubRoutes.tsx | 4 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 2 +-
.../{routes.types.tsx => types/datanode.types.tsx} | 10 +-
.../src/{ => types}/routes.types.tsx | 0
.../recon/ozone-recon-web/src/utils/themeIcons.tsx | 89 +
.../src/views/Datanodes/Datanodes.less | 16 +-
.../src/views/Datanodes/Datanodes.tsx | 82 +-
.../src/views/Overview/Overview.tsx | 86 +-
.../src/views/Pipelines/Pipelines.tsx | 49 +-
.../webapps/recon/ozone-recon-web/yarn.lock | 13 +-
.../ozone/recon/AbstractOMMetadataManagerTest.java | 15 +-
...rKeyService.java => TestContainerEndpoint.java} | 173 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 408 +++++
.../ozone/recon/api/TestTaskStatusService.java | 8 +-
.../ozone/recon/api/TestUtilizationService.java | 77 -
.../ozone/recon/fsck/TestMissingContainerTask.java | 126 ++
.../TestReconInternalSchemaDefinition.java | 12 +-
.../scm/AbstractReconContainerManagerTest.java | 38 +-
.../ozone/recon/scm/TestReconContainerManager.java | 16 +
...TestReconIncrementalContainerReportHandler.java | 36 +-
...rManagerTest.java => TestReconNodeManager.java} | 69 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 7 +-
.../hadoop/ozone/recon/tasks/DummyReconDBTask.java | 2 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 125 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 108 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 55 +-
.../recon/types/GuiceInjectorUtilsForTests.java | 27 +-
...uthParser.java => AWSV4SignatureProcessor.java} | 204 ++-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 1 +
.../apache/hadoop/ozone/s3/HeaderPreprocessor.java | 16 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 96 +-
.../hadoop/ozone/s3/RootPageDisplayFilter.java | 61 +
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 4 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 2 +-
...{AWSAuthParser.java => SignatureProcessor.java} | 28 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 10 -
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 13 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 17 +-
.../s3/header/AuthenticationHeaderParser.java | 69 -
.../ozone/s3/header/AuthorizationHeaderV4.java | 30 +-
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 4 +-
.../ozone/s3/util/RangeHeaderParserUtil.java | 2 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 10 +-
.../ozone/s3/TestAWSV4SignatureProcessor.java | 103 ++
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 13 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 17 +-
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 30 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 27 +-
.../ozone/s3/header/TestAuthorizationHeaderV4.java | 2 +-
.../hadoop/ozone/freon/BaseAppendLogGenerator.java | 60 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 133 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 188 +++
...hunkGenerator.java => DatanodeBlockPutter.java} | 143 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 2 +-
.../freon/FollowerAppendLogEntryGenerator.java | 468 ++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 14 +-
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 2 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 23 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 24 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 21 +-
.../ozone/freon/OzoneClientKeyValidator.java | 9 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 12 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 6 +-
.../ozone/genesis/BenchmarkChunkManager.java | 181 +++
.../org/apache/hadoop/ozone/genesis/Genesis.java | 11 +
.../hadoop/ozone/segmentparser/BaseLogParser.java | 55 +
.../segmentparser/DatanodeRatisLogParser.java | 60 +
.../segmentparser/GenericRatisLogParser.java} | 33 +-
.../ozone/segmentparser/OMRatisLogParser.java | 46 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 50 +-
.../hadoop/ozone/segmentparser/package-info.java} | 9 +-
pom.xml | 207 +--
837 files changed, 21438 insertions(+), 7666 deletions(-)
diff --cc hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
index 7275a76,89472f2..adc6e0f
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
@@@ -19,53 -19,34 +19,55 @@@ Library OperatingSyste
Resource ../commonlib.robot
*** Variables ***
+${OfsBucket1} om:9862/fstest/bucket1
+${O3fsBucket1} bucket1.fstest/
+${O3Bucket1} o3://om/fstest/bucket1
+${O3Bucket2} o3://om/fstest/bucket2
+${O3Bucket3} o3://om/fstest2/bucket3
+${OfsNonExistBucket} om:9862/abc/def
+${O3fsNonExistBucket} def.abc/
+${NonExistVolume} abc
*** Test Cases ***
-Create volume and bucket
+Create volume and bucket for Ozone file System test
Execute ozone sh volume create o3://om/fstest --quota 100TB
+ Execute ozone sh bucket create ${O3Bucket1}
+ Execute ozone sh bucket create ${O3Bucket2}
+
Execute ozone sh volume create o3://om/fstest2 --quota 100TB
- Execute ozone sh bucket create o3://om/fstest/bucket1
- Execute ozone sh bucket create o3://om/fstest/bucket2
- Execute ozone sh bucket create o3://om/fstest2/bucket3
+ Execute ozone sh bucket create ${O3Bucket3}
+
+
+Check volume from o3fs
+ ${result} = Execute ozone fs -ls o3fs://${O3fsBucket1}
-Check volume from ozonefs
- ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
+Test ozone shell with ofs
+ Test ozone shell with scheme ofs om:9862/fstest/bucket1 om:9862/fstest/bucket2 om:9862/fstest2/bucket3 om:9862/abc/def
-Run ozoneFS tests
- Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
- ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name'
+Test ozone shell with o3fs
+ Test ozone shell with scheme o3fs bucket1.fstest/ bucket2.fstest/ bucket3.fstest2/ def.abc/
+
+*** Keywords ***
+Test ozone shell with scheme
+ [arguments] ${scheme} ${testBucket1} ${testBucket2} ${testBucket3} ${nonExistBucket}
+
+ ${result} = Execute ozone fs -ls ${scheme}://${testBucket1}
+ Execute ozone fs -mkdir -p ${scheme}://${testBucket1}/testdir/deep
+ ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name'
Should contain ${result} testdir/deep
- Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
- ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name'
+
+ Execute ozone fs -copyFromLocal NOTICE.txt ${scheme}://${testBucket1}/testdir/deep/
+ ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name'
Should contain ${result} NOTICE.txt
+ ${result} = Execute ozone sh key info o3://om/fstest/bucket1/testdir/deep/NOTICE.txt | jq -r '.replicationFactor'
+ Should Be Equal ${result} 3
- Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
- ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name'
+ Execute ozone fs -put NOTICE.txt ${scheme}://${testBucket1}/testdir/deep/PUTFILE.txt
+ ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name'
Should contain ${result} PUTFILE.txt
- ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+ ${result} = Execute ozone fs -ls ${scheme}://${testBucket1}/testdir/deep/
Should contain ${result} NOTICE.txt
Should contain ${result} PUTFILE.txt
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org