You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2020/10/25 17:23:47 UTC
[hadoop-ozone] 01/11: Merge branch 'master' into HDDS-2823
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6de98c6d124ec127310e1aa9fac0cbc576991cfc
Merge: 40127b3 10df489
Author: Nandakumar <na...@apache.org>
AuthorDate: Sat Oct 24 20:07:04 2020 +0530
Merge branch 'master' into HDDS-2823
.github/close-pending.sh | 41 +
.github/closing-message.txt | 7 +
.github/comment-commands/close.sh | 10 +-
.github/comment-commands/pending.sh | 1 +
.github/workflows/close-pending.yaml | 32 +
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 411 ++--
LICENSE.txt | 4 +-
dev-support/byteman/appendlog.btm | 28 +
dev-support/byteman/hcfs-read.btm | 67 +
dev-support/byteman/hcfs-write.btm | 111 ++
dev-support/byteman/ratis-flush.btm | 34 +
dev-support/byteman/ratis-no-flush.btm | 25 +
dev-support/byteman/watchforcommit.btm | 35 +
dev-support/byteman/watchforcommit_all.btm | 47 +
hadoop-hdds/client/pom.xml | 15 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 38 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 48 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 40 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 11 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 8 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 62 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 158 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 49 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 10 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 37 +-
.../hdds/scm/storage/DummyChunkInputStream.java | 2 +-
.../storage/TestBlockOutputStreamCorrectness.java | 224 +++
.../hadoop/hdds/scm/storage/TestBufferPool.java | 46 +
hadoop-hdds/common/pom.xml | 14 +-
hadoop-hdds/common/src/main/conf/hadoop-env.sh | 13 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 21 +
.../hadoop/hdds/cli/SubcommandWithParent.java | 30 +
.../org/apache/hadoop/hdds/cli/package-info.java | 4 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 240 ++-
.../hadoop/hdds/protocol/DatanodeDetails.java | 159 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 160 +-
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 110 +-
.../RequestTypeDependentRetryPolicyCreator.java | 120 ++
.../retrypolicy/RetryLimitedPolicyCreator.java | 47 +
.../hdds/ratis/retrypolicy/RetryPolicyCreator.java | 29 +
.../hdds/ratis/retrypolicy/package-info.java | 23 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 4 +
.../hadoop/hdds/scm/ByteStringConversion.java | 18 +-
.../hadoop/hdds/scm/PipelineChoosePolicy.java | 37 +
.../hdds/scm/PipelineRequestInformation.java | 59 +
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 23 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 15 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 2 +
.../hadoop/hdds/scm/exceptions/SCMException.java | 4 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 2 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 42 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 229 ++-
.../x509/certificate/utils/CertificateCodec.java | 2 +-
.../hadoop/hdds/utils/BackgroundService.java | 73 +-
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../hadoop/hdds/utils/BackgroundTaskQueue.java | 5 +-
.../apache/hadoop/hdds/utils/RatisVersionInfo.java | 61 +
.../org/apache/hadoop/hdds/utils/Scheduler.java | 2 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 9 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 56 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 14 +-
.../common/ChunkBufferImplWithByteBuffer.java | 10 +-
.../org/apache/hadoop/ozone/common/Storage.java | 2 +-
.../container/common/helpers/ChunkInfoList.java | 56 +
.../common/src/main/resources/ozone-default.xml | 82 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 13 +
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 7 +-
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +-
.../hdds/ratis/conf/TestRaftClientConfig.java | 62 +
.../hdds/ratis/conf/TestRatisClientConfig.java | 68 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 29 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 62 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 16 +-
hadoop-hdds/config/pom.xml | 4 +-
.../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 57 +
.../hadoop/hdds/conf/InMemoryConfiguration.java | 58 +
.../hdds/conf/TestConfigurationReflectionUtil.java | 111 ++
hadoop-hdds/container-service/pom.xml | 11 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 18 +-
.../container/common/helpers/ContainerUtils.java | 66 +-
.../container/common/impl/ContainerDataYaml.java | 3 +
.../container/common/impl/HddsDispatcher.java | 61 +-
.../common/impl/StorageLocationReport.java | 11 +-
.../container/common/interfaces/BlockIterator.java | 5 +-
.../container/common/interfaces/Container.java | 7 -
.../common/statemachine/DatanodeStateMachine.java | 51 +-
.../common/statemachine/SCMConnectionManager.java | 9 +-
.../common/statemachine/StateContext.java | 31 +
.../CloseContainerCommandHandler.java | 2 +-
.../CreatePipelineCommandHandler.java | 15 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 92 +-
.../states/datanode/RunningDatanodeState.java | 14 +-
.../states/endpoint/RegisterEndpointTask.java | 6 +-
.../common/transport/server/XceiverServerSpi.java | 22 +-
.../server/ratis/ContainerStateMachine.java | 14 +-
.../transport/server/ratis/XceiverServerRatis.java | 93 +-
.../container/common/utils/ContainerCache.java | 95 +-
.../container/common/utils/HddsVolumeUtil.java | 7 +-
.../container/common/utils/ReferenceCountedDB.java | 10 +-
.../container/common/volume/MutableVolumeSet.java | 10 +-
.../container/keyvalue/KeyValueBlockIterator.java | 156 --
.../container/keyvalue/KeyValueContainer.java | 29 +-
.../container/keyvalue/KeyValueContainerCheck.java | 12 +-
.../container/keyvalue/KeyValueContainerData.java | 48 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 18 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 184 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 124 +-
.../keyvalue/interfaces/BlockManager.java | 12 +
.../background/BlockDeletingService.java | 75 +-
.../metadata/AbstractDatanodeDBDefinition.java | 74 +
.../container/metadata/AbstractDatanodeStore.java | 297 +++
.../ozone/container/metadata/BlockDataCodec.java | 47 +
.../container/metadata/ChunkInfoListCodec.java | 45 +
.../metadata/DatanodeSchemaOneDBDefinition.java | 91 +
.../metadata/DatanodeSchemaTwoDBDefinition.java | 81 +
.../ozone/container/metadata/DatanodeStore.java | 94 +
.../metadata/DatanodeStoreSchemaOneImpl.java | 49 +
.../metadata/DatanodeStoreSchemaTwoImpl.java | 44 +
.../ozone/container/metadata/DatanodeTable.java | 130 ++
.../metadata/SchemaOneChunkInfoListCodec.java | 68 +
.../metadata/SchemaOneDeletedBlocksTable.java | 180 ++
.../container/metadata/SchemaOneKeyCodec.java | 106 ++
.../ozone/container/metadata/package-info.java | 22 +
.../ozone/container/ozoneimpl/ContainerReader.java | 2 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 30 +-
.../replication/GrpcReplicationClient.java | 29 +-
.../replication/SimpleContainerDownloader.java | 30 +-
.../protocol/StorageContainerDatanodeProtocol.java | 14 +-
.../protocol/commands/CreatePipelineCommand.java | 46 +-
...inerDatanodeProtocolClientSideTranslatorPB.java | 12 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 4 +-
.../main/resources/webapps/hddsDatanode/index.html | 4 +-
.../ozone/container/ContainerTestHelper.java | 5 +
.../hadoop/ozone/container/common/ScmTestMock.java | 9 +-
.../container/common/TestBlockDeletingService.java | 113 +-
.../ozone/container/common/TestContainerCache.java | 73 +-
.../common/TestKeyValueContainerData.java | 3 +
.../TestSchemaOneBackwardsCompatibility.java | 626 +++++++
.../common/impl/TestContainerDataYaml.java | 3 +
.../common/statemachine/TestStateContext.java | 85 +
.../TestCreatePipelineCommandHandler.java | 7 +-
.../states/datanode/TestRunningDatanodeState.java | 90 +
.../keyvalue/TestKeyValueBlockIterator.java | 340 ++--
.../container/keyvalue/TestKeyValueContainer.java | 75 +-
.../keyvalue/TestKeyValueContainerCheck.java | 18 +-
.../container/ozoneimpl/TestContainerReader.java | 128 +-
.../container/ozoneimpl/TestOzoneContainer.java | 65 +-
.../test/resources/123-dn-container.db/000024.sst | Bin 0 -> 1022 bytes
.../test/resources/123-dn-container.db/000026.sst | Bin 0 -> 827 bytes
.../test/resources/123-dn-container.db/000032.sst | Bin 0 -> 896 bytes
.../test/resources/123-dn-container.db/000034.log | 0
.../src/test/resources/123-dn-container.db/CURRENT | 1 +
.../test/resources/123-dn-container.db/IDENTITY | 1 +
.../resources/123-dn-container.db/MANIFEST-000033 | Bin 0 -> 297 bytes
.../resources/123-dn-container.db/OPTIONS-000033 | 165 ++
.../resources/123-dn-container.db/OPTIONS-000036 | 165 ++
.../src/test/resources/123.container | 10 +
hadoop-hdds/docs/content/_index.md | 5 +-
hadoop-hdds/docs/content/_index.zh.md | 2 +-
hadoop-hdds/docs/content/beyond/Containers.md | 234 ---
hadoop-hdds/docs/content/beyond/Containers.zh.md | 203 --
.../docs/content/beyond/DockerCheatSheet.md | 88 -
.../docs/content/beyond/DockerCheatSheet.zh.md | 85 -
hadoop-hdds/docs/content/beyond/_index.md | 30 -
hadoop-hdds/docs/content/beyond/_index.zh.md | 27 -
hadoop-hdds/docs/content/concept/Containers.md | 47 +
hadoop-hdds/docs/content/concept/Containers.png | Bin 0 -> 24775 bytes
hadoop-hdds/docs/content/concept/Datanodes.md | 5 +-
hadoop-hdds/docs/content/concept/Datanodes.zh.md | 3 +
hadoop-hdds/docs/content/concept/Hdds.md | 52 -
hadoop-hdds/docs/content/concept/Hdds.zh.md | 40 -
hadoop-hdds/docs/content/concept/Overview.md | 7 +-
hadoop-hdds/docs/content/concept/Overview.zh.md | 7 +-
.../docs/content/concept/OzoneManager-ReadPath.png | Bin 0 -> 81030 bytes
.../content/concept/OzoneManager-WritePath.png | Bin 0 -> 96696 bytes
hadoop-hdds/docs/content/concept/OzoneManager.md | 63 +-
hadoop-hdds/docs/content/concept/OzoneManager.png | Bin 0 -> 13327 bytes
.../docs/content/concept/OzoneManager.zh.md | 9 +
.../content/concept/StorageContainerManager.md | 99 +
.../content/concept/StorageContainerManager.png | Bin 0 -> 13336 bytes
.../content/concept/StorageContainerManager.zh.md | 49 +
hadoop-hdds/docs/content/concept/_index.md | 4 +-
hadoop-hdds/docs/content/concept/_index.zh.md | 2 +-
hadoop-hdds/docs/content/design/ec.md | 39 +
hadoop-hdds/docs/content/design/multiraft.md | 2 +-
.../docs/content/design/namespace-support.md | 6 +-
hadoop-hdds/docs/content/design/ofs.md | 131 --
.../content/design/ozone-enhancement-proposals.md | 2 +-
hadoop-hdds/docs/content/design/recon2.md | 2 +-
hadoop-hdds/docs/content/design/scmha.md | 4 +-
hadoop-hdds/docs/content/design/storage-class.md | 28 +
hadoop-hdds/docs/content/design/topology.md | 29 +
hadoop-hdds/docs/content/design/trash.md | 7 +-
hadoop-hdds/docs/content/design/typesafeconfig.md | 10 +-
...e-volume-management.md => volume-management.md} | 0
hadoop-hdds/docs/content/feature/GDPR.md | 80 +
hadoop-hdds/docs/content/feature/GDPR.zh.md | 41 +
.../docs/content/feature/HA-OM-doublebuffer.png | Bin 0 -> 77661 bytes
hadoop-hdds/docs/content/feature/HA-OM.png | Bin 0 -> 60888 bytes
hadoop-hdds/docs/content/feature/HA.md | 115 ++
hadoop-hdds/docs/content/feature/Observability.md | 224 +++
hadoop-hdds/docs/content/feature/Recon.md | 47 +
hadoop-hdds/docs/content/feature/Topology.md | 108 ++
hadoop-hdds/docs/content/feature/_index.md | 34 +
.../docs/content/{gdpr => feature}/_index.zh.md | 0
hadoop-hdds/docs/content/gdpr/GDPR in Ozone.md | 42 -
hadoop-hdds/docs/content/gdpr/GDPR in Ozone.zh.md | 36 -
hadoop-hdds/docs/content/gdpr/_index.md | 38 -
hadoop-hdds/docs/content/interface/CSI.md | 22 +-
hadoop-hdds/docs/content/interface/CSI.png | Bin 0 -> 27210 bytes
hadoop-hdds/docs/content/interface/CSI.zh.md | 92 +
hadoop-hdds/docs/content/interface/Cli.md | 208 +++
hadoop-hdds/docs/content/interface/JavaApi.md | 5 +-
hadoop-hdds/docs/content/interface/JavaApi.zh.md | 3 +
hadoop-hdds/docs/content/interface/O3fs.md | 127 ++
hadoop-hdds/docs/content/interface/O3fs.zh.md | 167 ++
hadoop-hdds/docs/content/interface/Ofs.md | 227 +++
hadoop-hdds/docs/content/interface/OzoneFS.md | 167 --
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 159 --
hadoop-hdds/docs/content/interface/S3.md | 29 +-
hadoop-hdds/docs/content/interface/S3.zh.md | 3 +
hadoop-hdds/docs/content/interface/_index.md | 4 +-
hadoop-hdds/docs/content/recipe/Prometheus.md | 5 +-
hadoop-hdds/docs/content/recipe/Prometheus.zh.md | 7 +-
hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 1 -
.../docs/content/security/SecuityWithRanger.md | 43 -
.../docs/content/security/SecuityWithRanger.zh.md | 35 -
hadoop-hdds/docs/content/security/SecureOzone.md | 3 +
.../docs/content/security/SecureOzone.zh.md | 6 +-
.../docs/content/security/SecuringDatanodes.md | 39 +-
.../docs/content/security/SecuringDatanodes.zh.md | 53 +
.../docs/content/security/SecuringOzoneHTTP.md | 7 +-
hadoop-hdds/docs/content/security/SecuringS3.md | 5 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 3 +
hadoop-hdds/docs/content/security/SecuringTDE.md | 9 +-
.../docs/content/security/SecuringTDE.zh.md | 55 +
hadoop-hdds/docs/content/security/SecurityAcls.md | 11 +-
.../docs/content/security/SecurityAcls.zh.md | 69 +
.../docs/content/security/SecurityWithRanger.md | 46 +
.../docs/content/security/SecurityWithRanger.zh.md | 38 +
hadoop-hdds/docs/content/shell/BucketCommands.md | 100 -
.../docs/content/shell/BucketCommands.zh.md | 98 -
hadoop-hdds/docs/content/shell/Format.md | 69 -
hadoop-hdds/docs/content/shell/Format.zh.md | 65 -
hadoop-hdds/docs/content/shell/KeyCommands.md | 177 --
hadoop-hdds/docs/content/shell/KeyCommands.zh.md | 176 --
hadoop-hdds/docs/content/shell/VolumeCommands.md | 114 --
.../docs/content/shell/VolumeCommands.zh.md | 108 --
hadoop-hdds/docs/content/shell/_index.md | 28 -
hadoop-hdds/docs/content/shell/_index.zh.md | 27 -
hadoop-hdds/docs/content/start/FromSource.md | 38 +-
hadoop-hdds/docs/content/start/FromSource.zh.md | 7 +-
hadoop-hdds/docs/content/start/OnPrem.md | 2 +-
hadoop-hdds/docs/content/start/OnPrem.zh.md | 2 +-
.../docs/content/start/StartFromDockerHub.md | 6 +-
hadoop-hdds/docs/content/tools/TestTools.md | 14 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 14 +-
hadoop-hdds/docs/content/tools/_index.md | 6 +-
hadoop-hdds/docs/pom.xml | 4 +-
.../themes/ozonedoc/layouts/_default/single.html | 2 +-
.../themes/ozonedoc/layouts/design/section.html | 2 +-
.../ozonedoc/layouts/partials/languages.html | 3 +-
.../themes/ozonedoc/layouts/partials/navbar.html | 6 +-
.../themes/ozonedoc/layouts/partials/sidebar.html | 14 +-
.../docs/themes/ozonedoc/static/css/ozonedoc.css | 23 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 7 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 32 +-
.../x509/certificate/authority/BaseApprover.java | 11 +-
.../certificate/authority/DefaultApprover.java | 11 +
.../certificate/authority/DefaultCAServer.java | 42 +-
.../certificate/client/DNCertificateClient.java | 5 +-
.../certificates/utils/CertificateSignRequest.java | 2 +-
.../certificates/utils/SelfSignedCertificate.java | 109 +-
.../server/OzoneProtocolMessageDispatcher.java | 41 +-
.../hadoop/hdds/server/http/HttpServer2.java | 21 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 29 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 42 +-
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 41 +-
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 8 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 41 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 17 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 40 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 105 ++
.../org/apache/hadoop/hdds/utils/db/Table.java | 69 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 59 +
.../resources/webapps/static/angular-1.7.9.min.js | 350 ----
.../resources/webapps/static/angular-1.8.0.min.js | 350 ++++
.../webapps/static/angular-route-1.7.9.min.js | 17 -
.../webapps/static/angular-route-1.8.0.min.js | 17 +
.../x509/certificates/TestRootCertificate.java | 42 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 5 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 10 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 61 +
hadoop-hdds/hadoop-dependency-client/README.md | 18 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 8 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/{proto => resources}/proto.lock | 0
hadoop-hdds/interface-client/pom.xml | 4 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 16 +-
.../interface-client/src/main/proto/proto.lock | 1938 -------------------
.../interface-client/src/main/resources/proto.lock | 1958 ++++++++++++++++++++
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 9 +-
.../src/main/proto/ScmServerProtocol.proto | 2 +
.../interface-server/src/main/proto/proto.lock | 1786 ------------------
.../interface-server/src/main/resources/proto.lock | 1786 ++++++++++++++++++
hadoop-hdds/pom.xml | 20 +-
hadoop-hdds/server-scm/pom.xml | 5 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 14 +-
.../block/DatanodeDeletedBlockTransactions.java | 32 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 24 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 7 +-
.../container/AbstractContainerReportHandler.java | 101 +-
.../hdds/scm/container/ContainerReplica.java | 43 +-
.../hdds/scm/container/ContainerReportHandler.java | 14 +-
.../hdds/scm/container/ContainerStateManager.java | 8 +-
.../IncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 133 +-
.../hdds/scm/container/SCMContainerManager.java | 70 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 55 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 6 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 30 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 68 +
.../scm/pipeline/BackgroundPipelineCreator.java | 35 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 4 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 42 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 39 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 40 +-
.../algorithms/HealthyPipelineChoosePolicy.java | 46 +
.../algorithms/PipelineChoosePolicyFactory.java | 106 ++
.../algorithms/RandomPipelineChoosePolicy.java | 38 +
.../pipeline/choose/algorithms/package-info.java | 18 +
.../algorithms/DefaultLeaderChoosePolicy.java | 42 +
.../choose/algorithms/LeaderChoosePolicy.java | 55 +
.../algorithms/LeaderChoosePolicyFactory.java | 75 +
.../algorithms/MinLeaderCountChoosePolicy.java | 91 +
.../leader/choose/algorithms/package-info.java | 19 +
.../SCMSecurityProtocolServerSideTranslatorPB.java | 17 +-
...lockLocationProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 17 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 4 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 6 +
.../hdds/scm/server/StorageContainerManager.java | 31 +-
.../src/main/resources/webapps/scm/index.html | 4 +-
.../main/resources/webapps/scm/scm-overview.html | 27 +-
.../src/main/resources/webapps/scm/scm.js | 4 -
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 47 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 135 ++
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 17 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 50 +
.../scm/container/TestContainerReportHandler.java | 197 +-
.../hdds/scm/container/TestReplicationManager.java | 92 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 85 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 161 ++
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 180 ++
.../TestPipelineChoosePolicyFactory.java | 94 +
.../choose/algorithms/TestLeaderChoosePolicy.java | 74 +
.../ozone/container/common/TestEndPoint.java | 2 +-
.../testutils/ReplicationNodeManagerMock.java | 15 +
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 12 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 67 +
.../org/apache/hadoop/hdds/cli/package-info.java | 22 +
.../hdds/scm/cli/ReplicationManagerCommands.java | 23 +-
.../scm/cli/ReplicationManagerStartSubcommand.java | 21 +-
.../cli/ReplicationManagerStatusSubcommand.java | 32 +-
.../scm/cli/ReplicationManagerStopSubcommand.java | 25 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 +-
.../hadoop/hdds/scm/cli/SafeModeCommands.java | 27 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 22 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 72 +
.../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 43 +
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 65 +-
.../hdds/scm/cli/container/CloseSubcommand.java | 20 +-
.../hdds/scm/cli/container/ContainerCommands.java | 21 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 26 +-
.../hdds/scm/cli/container/DeleteSubcommand.java | 20 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 48 +-
.../hdds/scm/cli/container/ListSubcommand.java | 32 +-
.../hdds/scm/cli/container/WithScmClient.java | 29 -
.../hdds/scm/cli/datanode/DatanodeCommands.java | 21 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 48 +-
.../cli/pipeline/ActivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 48 +-
.../cli/pipeline/DeactivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 40 +-
.../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +-
hadoop-ozone/client/pom.xml | 4 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 72 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 3 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 126 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 8 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 103 +-
.../org/apache/hadoop/ozone/client/VolumeArgs.java | 45 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 30 +-
.../client/io/BlockOutputStreamEntryPool.java | 40 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 98 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 60 +-
.../hadoop/ozone/client/io/OzoneInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 29 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 117 +-
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 75 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 6 +-
.../apache/hadoop/ozone/freon/OzoneGetConf.java | 278 ---
.../apache/hadoop/ozone/freon/package-info.java | 21 -
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../hadoop/ozone/om/exceptions/OMException.java | 11 +-
.../ozone/om/exceptions/OMNotLeaderException.java | 2 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 253 ++-
.../ozone/om/helpers/BucketEncryptionKeyInfo.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 214 ++-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 24 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 111 +-
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 215 ---
.../hadoop/ozone/om/helpers/OmRenameKeys.java | 59 +
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 30 +
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 18 +
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 9 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 15 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 175 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 44 +-
.../ozone/security/OzoneTokenIdentifier.java | 61 +-
.../org/apache/hadoop/ozone/util/ExitManager.java | 33 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 15 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 49 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 22 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 59 +
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 4 +-
.../hadoop/ozone/om/helpers/TestOzoneFsUtils.java | 39 +
hadoop-ozone/csi/pom.xml | 4 +-
.../csi/src/main/{proto => resources}/proto.lock | 0
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 2 +
hadoop-ozone/dev-support/checks/bats.sh | 35 +
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 7 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 36 +
hadoop-ozone/dev-support/intellij/core-site.xml | 27 +
hadoop-ozone/dist/README.md | 52 +-
.../dist/dev-support/bin/dist-layout-stitching | 5 +
hadoop-ozone/dist/pom.xml | 13 +-
hadoop-ozone/dist/src/main/compose/failing1/.env | 1 +
.../src/main/compose/failing1/docker-compose.yaml | 1 +
.../dist/src/main/compose/failing1/docker-config | 1 +
.../dist/src/main/compose/failing1/test.sh | 36 +
hadoop-ozone/dist/src/main/compose/failing2/.env | 1 +
.../src/main/compose/failing2/docker-compose.yaml | 1 +
.../dist/src/main/compose/failing2/docker-config | 1 +
.../dist/src/main/compose/failing2/test.sh | 36 +
.../dist/src/main/compose/ozone-csi/docker-config | 3 +
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 19 +
.../src/main/compose/ozone-ha/docker-compose.yaml | 93 +
.../dist/src/main/compose/ozone-ha/docker-config | 35 +
.../dist/src/main/compose/ozone-ha/test.sh | 33 +
.../dist/src/main/compose/ozone-mr/common-config | 3 +-
.../main/compose/ozone-mr/hadoop27/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +-
.../main/compose/ozone-mr/hadoop31/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 7 +-
.../main/compose/ozone-mr/hadoop32/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop32/test.sh | 7 +-
.../dist/src/main/compose/ozone-mr/test.sh | 37 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 5 +-
.../dist/src/main/compose/ozone-om-ha-s3/test.sh | 2 +
.../src/main/compose/ozone-om-ha/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 7 +-
.../dist/src/main/compose/ozone-topology/test.sh | 4 +-
.../dist/src/main/compose/ozone/docker-config | 7 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 20 +-
.../src/main/compose/ozoneblockade/docker-config | 2 +
.../src/main/compose/ozones3-haproxy/docker-config | 3 +
.../src/main/compose/ozonesecure-mr/docker-config | 4 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 5 +-
.../main/compose/ozonesecure-om-ha/docker-config | 11 +-
.../src/main/compose/ozonesecure-om-ha/test.sh | 2 +
.../src/main/compose/ozonesecure/docker-config | 9 +-
.../dist/src/main/compose/ozonesecure/test.sh | 17 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 28 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 105 +-
hadoop-ozone/dist/src/main/compose/upgrade/.env | 21 +
.../dist/src/main/compose/upgrade/README.md | 29 +
.../src/main/compose/upgrade/docker-compose.yaml | 127 ++
.../dist/src/main/compose/upgrade/docker-config | 34 +
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 69 +
.../src/main/compose/upgrade/versions/README.md | 15 +
.../main/compose/upgrade/versions/ozone-0.5.0.sh | 19 +
.../main/compose/upgrade/versions/ozone-1.0.0.sh | 19 +
hadoop-ozone/dist/src/main/k8s/.gitignore | 15 +
.../src/main/k8s/definitions/ozone/config.yaml | 1 +
.../k8s/definitions/ozone/definitions/onenode.yaml | 2 +-
.../main/k8s/definitions/ozone/freon/freon.yaml | 2 +-
.../{pv-test => test-webserver}/flekszible.yaml | 0
.../webserver-deployment.yaml | 0
.../webserver-service.yaml | 0
.../webserver-volume.yaml | 0
.../examples/getting-started/config-configmap.yaml | 1 +
.../getting-started/datanode-statefulset.yaml | 10 -
.../getting-started/freon/freon-deployment.yaml | 2 +-
.../src/main/k8s/examples/getting-started/test.sh | 39 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../examples/minikube/freon/freon-deployment.yaml | 2 +-
.../dist/src/main/k8s/examples/minikube/test.sh | 39 +
.../src/main/k8s/examples/ozone-dev/Flekszible | 4 +-
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 2 +-
.../examples/ozone-dev/datanode-statefulset.yaml | 10 -
.../examples/ozone-dev/freon/freon-deployment.yaml | 2 +-
.../dist/src/main/k8s/examples/ozone-dev/test.sh | 39 +
.../dist/src/main/k8s/examples/ozone/Flekszible | 2 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../ozone/csi/csi-provisioner-deployment.yaml | 2 +-
.../k8s/examples/ozone/freon/freon-deployment.yaml | 2 +-
.../dist/src/main/k8s/examples/ozone/test.sh | 39 +
.../dist/src/main/k8s/examples/test-all.sh | 49 +
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 144 ++
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +-
.../dist/src/main/smoketest/admincli/admin.robot | 32 +
.../src/main/smoketest/admincli/container.robot | 73 +
.../src/main/smoketest/admincli/datanode.robot | 19 +-
.../src/main/smoketest/admincli/pipeline.robot | 49 +-
.../smoketest/admincli/replicationmanager.robot | 53 +
.../src/main/smoketest/admincli/safemode.robot | 45 +
.../main/smoketest/auditparser/auditparser.robot | 2 +-
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../dist/src/main/smoketest/basic/getconf.robot | 46 +
.../dist/src/main/smoketest/basic/links.robot | 152 ++
.../src/main/smoketest/basic/ozone-shell-lib.robot | 179 ++
.../main/smoketest/basic/ozone-shell-single.robot | 27 +
.../src/main/smoketest/basic/ozone-shell.robot | 121 +-
.../dist/src/main/smoketest/commonlib.robot | 34 +-
.../dist/src/main/smoketest/createbucketenv.robot | 2 +-
.../dist/src/main/smoketest/createmrenv.robot | 15 +-
.../src/main/smoketest/debug/ozone-debug.robot | 6 +-
.../dist/src/main/smoketest/failing/test1.robot | 21 +
.../dist/src/main/smoketest/failing/test2.robot | 21 +
.../dist/src/main/smoketest/freon/freon.robot | 2 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 2 +-
hadoop-ozone/dist/src/main/smoketest/lib/os.robot | 49 +
.../dist/src/main/smoketest/lib/os_tests.robot | 38 +
.../dist/src/main/smoketest/mapreduce.robot | 11 +-
.../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozone-lib/shell.robot | 53 +
.../src/main/smoketest/ozone-lib/shell_tests.robot | 58 +
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 12 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 20 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 18 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 133 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 12 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 40 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 23 +-
.../dist/src/main/smoketest/s3/objectdelete.robot | 28 +-
.../src/main/smoketest/s3/objectmultidelete.robot | 24 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 40 +-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 47 +
.../smoketest/security/bucket-encryption.robot | 45 +
.../main/smoketest/security/ozone-secure-s3.robot | 15 +
.../smoketest/security/ozone-secure-token.robot | 16 +-
.../dist/src/main/smoketest/spnego/web.robot | 16 +-
.../src/main/smoketest/topology/loaddata.robot | 2 +-
hadoop-ozone/dist/src/shell/hdds/hadoop-daemons.sh | 2 +-
.../dist/src/shell/hdds/hadoop-functions.sh | 6 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 22 +-
hadoop-ozone/dist/src/shell/ozone/stop-ozone.sh | 8 +-
hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 23 +
.../src/shell/upgrade/1.0.0/01-migrate-scm-db.sh | 24 +
.../dist/src/test/shell/compose_testlib.bats | 37 +
hadoop-ozone/dist/src/test/shell/gc_opts.bats | 6 +-
hadoop-ozone/dist/src/test/shell/k8s_testlib.bats | 55 +
hadoop-ozone/dist/src/test/shell/test1/test.sh | 15 +
hadoop-ozone/dist/src/test/shell/test2/test.sh | 17 +
.../dist/src/test/shell/test3/subtest1/test.sh | 17 +
hadoop-ozone/dist/src/test/shell/test4/test.sh | 17 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 40 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../fault-injection-test/network-tests/pom.xml | 2 +-
.../src/test/blockade/ozone/client.py | 10 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
.../hadoop/ozone/insight/BaseInsightPoint.java | 34 +-
.../ozone/insight/BaseInsightSubCommand.java | 6 +
.../org/apache/hadoop/ozone/insight/Insight.java | 4 +
.../apache/hadoop/ozone/insight/InsightPoint.java | 4 +-
.../hadoop/ozone/insight/MetricGroupDisplay.java | 4 +-
.../hadoop/ozone/insight/MetricsSubCommand.java | 33 +-
.../datanode/DatanodeDispatcherInsight.java | 107 ++
.../insight/datanode/PipelineComponentUtil.java | 78 +
.../ozone/insight/datanode/RatisInsight.java | 45 +-
.../hadoop/ozone/insight/om/KeyManagerInsight.java | 2 +-
.../hadoop/ozone/insight/om/OmProtocolInsight.java | 2 +-
.../ozone/insight/scm/NodeManagerInsight.java | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
.../scm/ScmProtocolBlockLocationInsight.java | 2 +-
.../scm/ScmProtocolContainerLocationInsight.java | 2 +-
.../insight/scm/ScmProtocolDatanodeInsight.java | 2 +-
.../insight/scm/ScmProtocolSecurityInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 391 ++++
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 26 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 182 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 253 ++-
.../hadoop/fs/ozone/contract/OzoneContract.java | 18 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 25 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 216 +++
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 20 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 13 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +-
.../hadoop/ozone/TestStorageContainerManager.java | 35 +-
.../ozone/TestStorageContainerManagerHelper.java | 31 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 16 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 32 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 32 +-
.../rpc/TestCloseContainerHandlingByClient.java | 55 -
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 78 +-
.../rpc/TestContainerReplicationEndToEnd.java | 19 +-
.../TestContainerStateMachineFailureOnRead.java | 27 +-
.../rpc/TestContainerStateMachineFailures.java | 47 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 28 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 186 ++
.../client/rpc/TestFailureHandlingByClient.java | 28 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 28 +-
.../ozone/client/rpc/TestKeyInputStream.java | 119 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 28 +-
.../rpc/TestOzoneClientRetriesOnException.java | 2 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 845 ++++++++-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 40 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 3 -
.../client/rpc/TestValidateBCSIDOnRestart.java | 36 +-
.../ozone/client/rpc/TestWatchForCommit.java | 35 +-
.../commandhandler/TestBlockDeletion.java | 171 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 138 --
.../container/ozoneimpl/TestRatisManager.java | 124 --
.../hadoop/ozone/freon/TestDataValidate.java | 18 +-
.../ozone/freon/TestFreonWithDatanodeRestart.java | 17 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 17 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 203 ++
.../ozone/freon/TestOzoneClientKeyGenerator.java | 2 -
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 18 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 3 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 32 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 215 ++-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 120 ++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 110 +-
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 235 ---
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 76 +
.../ozone/om/TestOzoneManagerListVolumes.java | 18 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 26 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 26 +-
.../ozone/recon/TestReconWithOzoneManager.java | 46 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 148 ++
.../hadoop/ozone/scm/TestContainerSmallFile.java | 20 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 6 +-
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 50 +-
.../src/test/resources/log4j.properties | 3 -
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OmClientProtocol.proto | 60 +
.../src/main/{proto => resources}/proto.lock | 0
.../dev-support/findbugsExcludeFile.xml | 21 +
hadoop-ozone/interface-storage/pom.xml | 103 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 385 ++++
.../ozone/om/codec/OMTransactionInfoCodec.java | 0
.../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 0
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 69 +
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 0
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 59 +
.../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 0
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 68 +
.../hadoop/ozone/om/codec/S3SecretValueCodec.java | 0
.../ozone/om/codec/TokenIdentifierCodec.java | 66 +
.../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 0
.../apache/hadoop/ozone/om/codec/package-info.java | 24 +
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 216 +++
.../hadoop/ozone/om/helpers/OzoneAclStorage.java | 63 +
.../ozone/om/helpers/OzoneAclStorageUtil.java | 62 +
.../hadoop/ozone/om/helpers/package-info.java | 24 +
.../org/apache/hadoop/ozone/om/package-info.java | 24 +
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 163 ++
.../apache/hadoop/ozone/om/ratis/package-info.java | 24 +
.../src/main/proto/OmStorageProtocol.proto | 60 +
.../ozone/om/codec/TestOMTransactionInfoCodec.java | 0
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 116 ++
.../om/codec/TestOmMultipartKeyInfoCodec.java | 0
.../ozone/om/codec/TestOmPrefixInfoCodec.java | 0
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 121 ++
.../ozone/om/codec/TestS3SecretValueCodec.java | 0
.../apache/hadoop/ozone/om/codec/package-info.java | 0
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 0
.../hadoop/ozone/om/helpers/package-info.java | 24 +
hadoop-ozone/ozone-manager/pom.xml | 10 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 116 +-
.../apache/hadoop/ozone/om/KeyDeletingService.java | 9 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 17 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 351 ++--
.../apache/hadoop/ozone/om/OMMetadataManager.java | 385 ----
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 37 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 70 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 45 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 744 ++++++--
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 111 ++
.../org/apache/hadoop/ozone/om/VolumeManager.java | 9 -
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 39 -
.../hadoop/ozone/om/codec/OMDBDefinition.java | 19 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 58 -
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 58 -
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 57 -
.../ozone/om/codec/TokenIdentifierCodec.java | 56 -
.../apache/hadoop/ozone/om/codec/package-info.java | 3 +
.../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 53 +-
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 153 --
.../ozone/om/ratis/OzoneManagerRatisServer.java | 18 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 26 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 40 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 94 +-
.../om/request/bucket/OMBucketCreateRequest.java | 52 +
.../request/bucket/OMBucketSetPropertyRequest.java | 64 +
.../bucket/acl/OMBucketRemoveAclRequest.java | 2 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +
.../ozone/om/request/file/OMFileCreateRequest.java | 57 +-
.../ozone/om/request/file/OMFileRequest.java | 2 +-
.../om/request/key/OMAllocateBlockRequest.java | 61 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 79 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 127 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 45 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 26 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 171 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 66 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++
.../om/request/key/OMTrashRecoverRequest.java | 7 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 2 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 2 +-
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 2 +-
.../key/acl/prefix/OMPrefixSetAclRequest.java | 2 +-
.../S3InitiateMultipartUploadRequest.java | 60 +-
.../multipart/S3MultipartUploadAbortRequest.java | 71 +-
.../S3MultipartUploadCommitPartRequest.java | 73 +-
.../S3MultipartUploadCompleteRequest.java | 86 +-
.../om/request/volume/OMVolumeCreateRequest.java | 6 +
.../om/request/volume/OMVolumeSetQuotaRequest.java | 52 +-
.../hadoop/ozone/om/response/CleanupTableInfo.java | 3 +-
.../om/response/file/OMFileCreateResponse.java | 10 +-
.../response/key/AbstractOMKeyDeleteResponse.java | 126 ++
.../om/response/key/OMAllocateBlockResponse.java | 18 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 18 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 20 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 73 +-
.../om/response/key/OMKeysDeleteResponse.java | 56 +-
.../om/response/key/OMKeysRenameResponse.java | 80 +
.../om/response/key/OMOpenKeysDeleteRequest.java | 192 ++
.../om/response/key/OMOpenKeysDeleteResponse.java | 72 +
.../multipart/S3MultipartUploadAbortResponse.java | 20 +-
.../S3MultipartUploadCommitPartResponse.java | 21 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 10 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 2 +-
.../main/resources/webapps/ozoneManager/index.html | 4 +-
.../main/resources/webapps/ozoneManager/main.html | 2 +
.../webapps/ozoneManager/om-overview.html | 26 +
.../resources/webapps/ozoneManager/ozoneManager.js | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 43 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 79 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 151 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/TestNormalizePaths.java | 109 ++
.../ozone/om/request/TestOMRequestUtils.java | 130 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 41 +-
.../request/file/TestOMDirectoryCreateRequest.java | 6 +
.../om/request/key/TestOMAllocateBlockRequest.java | 22 +-
.../om/request/key/TestOMKeyCreateRequest.java | 182 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 15 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +
.../om/request/key/TestOMKeysRenameRequest.java | 160 ++
.../request/key/TestOMOpenKeysDeleteRequest.java | 419 +++++
.../TestS3InitiateMultipartUploadRequest.java | 59 +-
.../s3/multipart/TestS3MultipartRequest.java | 17 +-
.../TestS3MultipartUploadCommitPartRequest.java | 2 +-
.../volume/TestOMVolumeSetOwnerRequest.java | 2 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 69 +-
.../bucket/TestOMBucketCreateResponse.java | 8 +
.../bucket/TestOMBucketDeleteResponse.java | 8 +
.../bucket/TestOMBucketSetPropertyResponse.java | 9 +
.../file/TestOMDirectoryCreateResponse.java | 8 +
.../response/key/TestOMAllocateBlockResponse.java | 27 +-
.../om/response/key/TestOMKeyCommitResponse.java | 19 +-
.../om/response/key/TestOMKeyCreateResponse.java | 23 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 27 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 3 +
.../om/response/key/TestOMKeysDeleteResponse.java | 38 +-
.../om/response/key/TestOMKeysRenameResponse.java | 131 ++
.../response/key/TestOMOpenKeysDeleteResponse.java | 185 ++
.../s3/multipart/TestS3MultipartResponse.java | 16 +-
.../TestS3MultipartUploadAbortResponse.java | 21 +-
.../security/TestOMDelegationTokenResponse.java | 8 +
.../volume/TestOMVolumeCreateResponse.java | 8 +
.../volume/TestOMVolumeDeleteResponse.java | 8 +
.../volume/TestOMVolumeSetOwnerResponse.java | 9 +
.../volume/TestOMVolumeSetQuotaResponse.java | 9 +
.../ozone/security/TestOzoneTokenIdentifier.java | 19 +
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 23 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 143 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 138 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 180 +-
.../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 4 +-
.../fs/ozone/RootedOzoneClientAdapterImpl.java | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 6 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 49 +
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 6 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 44 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 9 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java | 44 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/pom.xml | 33 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../recon/schema/ContainerSchemaDefinition.java | 4 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 2 +-
.../ozone/recon/schema/StatsSchemaDefinition.java | 14 +-
.../recon/schema/UtilizationSchemaDefinition.java | 23 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../ozone/recon/MetricsServiceProviderFactory.java | 86 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 8 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 13 +-
.../hadoop/ozone/recon/ReconSchemaManager.java | 2 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 50 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 47 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 47 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 2 +-
.../ozone/recon/api/MetricsProxyEndpoint.java | 118 ++
.../hadoop/ozone/recon/api/NodeEndpoint.java | 2 +
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 59 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 28 +
.../ozone/recon/codec/DatanodeDetailsCodec.java | 50 +
.../ozone/recon/codec/ReconNodeDBKeyCodec.java | 46 +
.../hadoop/ozone/recon/codec/package-info.java | 22 +
.../apache/hadoop/ozone/recon/metrics/Metric.java | 51 +
.../ozone/recon/scm/ReconContainerManager.java | 61 +-
.../recon/scm/ReconContainerReportHandler.java | 2 +-
.../hadoop/ozone/recon/scm/ReconDBDefinition.java | 38 -
.../ReconIncrementalContainerReportHandler.java | 6 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 55 +-
.../ozone/recon/scm/ReconSCMDBDefinition.java | 61 +
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../ozone/recon/spi/MetricsServiceProvider.java | 60 +
.../spi/impl/ContainerDBServiceProviderImpl.java | 60 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 54 +-
.../spi/impl/PrometheusServiceProviderImpl.java | 213 +++
.../recon/spi/impl/ReconContainerDBProvider.java | 28 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 71 +
.../ozone/recon/tasks/FileSizeCountTask.java | 4 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 92 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 10 +-
.../hadoop/ozone/recon/tasks/TableCountTask.java | 185 ++
.../webapps/recon/ozone-recon-web/api/db.json | 48 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 4 +-
.../ozone-recon-web/src/types/datanode.types.tsx | 4 +-
.../ozone-recon-web/src/utils/columnSearch.less | 36 +
.../ozone-recon-web/src/utils/columnSearch.tsx | 94 +
.../src/views/datanodes/datanodes.less | 1 +
.../src/views/datanodes/datanodes.tsx | 101 +-
.../src/views/pipelines/pipelines.tsx | 32 +-
.../hadoop/ozone/recon/ReconTestInjector.java | 3 +
.../apache/hadoop/ozone/recon/TestReconUtils.java | 6 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 151 +-
.../scm/AbstractReconContainerManagerTest.java | 51 +-
.../ozone/recon/scm/TestReconContainerManager.java | 95 +-
...TestReconIncrementalContainerReportHandler.java | 63 +
.../ozone/recon/scm/TestReconNodeManager.java | 20 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 8 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 9 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 54 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 6 +-
.../ozone/recon/tasks/TestTableCountTask.java | 178 ++
.../test/resources/prometheus-test-response.txt | 21 +
hadoop-ozone/s3gateway/pom.xml | 9 +-
.../hadoop/ozone/s3/AWSSignatureProcessor.java | 435 +++++
.../hadoop/ozone/s3/AWSV4SignatureProcessor.java | 426 -----
.../hadoop/ozone/s3/OzoneClientProducer.java | 8 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 6 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 8 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 69 +-
.../ozone/s3/header/AuthorizationHeaderV2.java | 2 +-
.../ozone/s3/header/AuthorizationHeaderV4.java | 4 +-
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 36 +-
.../apache/hadoop/ozone/s3/util/RangeHeader.java | 6 +-
.../hadoop/ozone/client/ObjectStoreStub.java | 5 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 6 +-
.../hadoop/ozone/s3/TestAWSSignatureProcessor.java | 141 ++
.../ozone/s3/TestAWSV4SignatureProcessor.java | 103 -
.../hadoop/ozone/s3/endpoint/TestBucketHead.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 2 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 15 +-
.../ozone/s3/header/TestAuthorizationHeaderV4.java | 2 +-
hadoop-ozone/tools/pom.xml | 18 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 120 --
.../ozone/admin/om/GetServiceRolesSubcommand.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 12 +-
.../org/apache/hadoop/ozone/conf/OzoneGetConf.java | 86 +
.../ozone/conf/OzoneManagersCommandHandler.java | 53 +
.../ozone/conf/PrintConfKeyCommandHandler.java | 52 +
.../StorageContainerManagersCommandHandler.java | 52 +
.../org/apache/hadoop/ozone/conf/package-info.java | 21 +
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 165 +-
.../hadoop/ozone/debug/ContainerChunkInfo.java | 21 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 22 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 131 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 81 +
.../org/apache/hadoop/ozone/debug/ListTables.java | 19 +-
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 11 +-
.../org/apache/hadoop/ozone/debug/RDBParser.java | 38 +-
.../apache/hadoop/ozone/debug/RocksDBUtils.java | 49 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 22 +
.../hadoop/ozone/freon/ContentGenerator.java | 31 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 244 +++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 55 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +-
.../ozone/freon/HadoopNestedDirGenerator.java | 27 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 53 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 2 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 12 +-
.../hadoop/ozone/shell/ClearSpaceQuotaOptions.java | 43 +
.../apache/hadoop/ozone/shell/OzoneAddress.java | 103 +-
.../org/apache/hadoop/ozone/shell/OzoneShell.java | 14 +-
.../hadoop/ozone/shell/SetSpaceQuotaOptions.java | 44 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 8 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 17 +-
.../ozone/shell/bucket/ClearQuotaHandler.java | 55 +
.../ozone/shell/bucket/CreateBucketHandler.java | 13 +
.../ozone/shell/bucket/LinkBucketHandler.java | 79 +
.../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 62 +
.../hadoop/ozone/shell/keys/KeyCommands.java | 12 +-
.../hadoop/ozone/shell/token/GetTokenHandler.java | 8 +-
.../ozone/shell/token/PrintTokenHandler.java | 3 +-
.../ozone/shell/token/RenewTokenHandler.java | 4 +-
.../hadoop/ozone/shell/token/TokenCommands.java | 16 +-
.../hadoop/ozone/shell/token/TokenOption.java | 38 +-
.../ozone/shell/volume/ClearQuotaHandler.java | 53 +
.../ozone/shell/volume/CreateVolumeHandler.java | 20 +-
.../hadoop/ozone/shell/volume/SetQuotaHandler.java | 61 +
.../ozone/shell/volume/UpdateVolumeHandler.java | 11 -
.../hadoop/ozone/shell/volume/VolumeCommands.java | 16 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../hadoop/ozone/conf/TestGetConfOptions.java | 90 +
.../ozone/debug/TestDBDefinitionFactory.java | 59 +
.../hadoop/ozone/freon/TestContentGenerator.java | 82 +
.../TestGenerateOzoneRequiredConfigurations.java | 5 +-
.../hadoop/ozone/shell/TestOzoneAddress.java | 6 +-
.../shell/TestOzoneAddressClientCreation.java | 172 ++
hadoop-ozone/upgrade/pom.xml | 57 -
.../org/apache/hadoop/ozone/upgrade/Balance.java | 38 -
.../org/apache/hadoop/ozone/upgrade/Execute.java | 37 -
.../hadoop/ozone/upgrade/InPlaceUpgrade.java | 45 -
.../java/org/apache/hadoop/ozone/upgrade/Plan.java | 38 -
.../apache/hadoop/ozone/upgrade/package-info.java | 23 -
pom.xml | 93 +-
999 files changed, 37896 insertions(+), 15714 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 8ed5ab6,a7aca16..03da6dd
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -365,11 -379,6 +379,13 @@@ public final class OzoneConsts
public static final String CONTAINER_DB_TYPE_ROCKSDB = "RocksDB";
public static final String CONTAINER_DB_TYPE_LEVELDB = "LevelDB";
+ // SCM HA
+ public static final String SCM_SERVICE_ID_DEFAULT = "scmServiceIdDefault";
+
+ // SCM Ratis snapshot file to store the last applied index
+ public static final String SCM_RATIS_SNAPSHOT_INDEX = "scmRatisSnapshotIndex";
+
+ public static final String SCM_RATIS_SNAPSHOT_TERM = "scmRatisSnapshotTerm";
+ // An on-disk transient marker file used when replacing DB with checkpoint
+ public static final String DB_TRANSIENT_MARKER = "dbInconsistentMarker";
}
diff --cc hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
index eecb512,653300e..816e1b9
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
@@@ -92,9 -94,9 +94,12 @@@ public final class ConfigurationReflect
forcedFieldSet(field, configuration,
from.getTimeDuration(key, "0s", configAnnotation.timeUnit()));
break;
+ case SIZE:
+ forcedFieldSet(field, configuration,
+ from.getStorageSize(key, "0B", configAnnotation.sizeUnit()));
+ case CLASS:
+ forcedFieldSet(field, configuration,
+ from.getClass(key, Object.class));
break;
default:
throw new ConfigurationException(
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9bf687d,3cf12e7..cbd713c
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -50,14 -42,8 +50,15 @@@ import org.apache.hadoop.hdds.conf.Conf
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+ import org.apache.hadoop.hdds.scm.PipelineChoosePolicy;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
+import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl;
+import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
+import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
+import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisSnapshotInfo;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.block.BlockManager;
@@@ -96,7 -82,8 +97,8 @@@ import org.apache.hadoop.hdds.scm.node.
import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
-import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl;
+ import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@@ -1175,56 -1119,6 +1180,56 @@@ public final class StorageContainerMana
return this.clusterMap;
}
+ private static SCMRatisServer initializeRatisServer(
+ OzoneConfiguration conf, StorageContainerManager scm) throws IOException {
+ SCMNodeDetails scmNodeDetails = SCMNodeDetails
+ .initStandAlone(conf);
+ //TODO enable Ratis group
+ SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer(
+ conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class),
+ scm, scmNodeDetails, Collections.EMPTY_LIST,
+ SCMRatisServer.getSCMRatisDirectory(conf));
+ if (scmRatisServer != null) {
+ LOG.info("SCM Ratis server initialized at port {}",
+ scmRatisServer.getServerPort());
+ } // TODO error handling for scmRatisServer creation failure
+ return scmRatisServer;
+ }
+
+ @VisibleForTesting
+ public SCMRatisServer getScmRatisServer() {
+ return scmRatisServer;
+ }
+
+ public void setScmRatisServer(SCMRatisServer scmRatisServer) {
+ this.scmRatisServer = scmRatisServer;
+ }
+
+ @VisibleForTesting
+ public SCMRatisSnapshotInfo getSnapshotInfo() {
+ return scmRatisSnapshotInfo;
+ }
+
+ @VisibleForTesting
+ public long getRatisSnapshotIndex() {
+ return scmRatisSnapshotInfo.getIndex();
+ }
+
+ /**
+ * Save ratis snapshot to SCM meta store and local disk.
+ */
+ public TermIndex saveRatisSnapshot() throws IOException {
+ TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex();
+ if (scmMetadataStore != null) {
+ // Flush the SCM state to disk
- scmMetadataStore.getStore().flush();
++ scmMetadataStore.getStore().flushDB();
+ }
+
+ scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex);
+
+ return snapshotIndex;
+ }
+
/**
* Get the safe mode status of all rules.
*
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 597a317,2b492a2..90f4996
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@@ -26,9 -26,6 +26,12 @@@ import static org.apache.hadoop.hdds.Hd
import static org.apache.hadoop.hdds.HddsConfigKeys
.HDDS_SCM_SAFEMODE_PIPELINE_CREATION;
import static org.junit.Assert.fail;
++<<<<<<< HEAD
+
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.junit.Ignore;
++=======
++>>>>>>> master
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java
index 18b9c4c,7a28ba5..3ac5ad8
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java
@@@ -74,15 -74,13 +74,12 @@@ public class TestFreonWithDatanodeResta
ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3));
conf.setFromObject(ratisServerConfig);
- conf.setTimeDuration(
- RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." +
- "rpc.request.timeout",
- 3, TimeUnit.SECONDS);
- conf.setTimeDuration(
- RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." +
- "watch.request.timeout",
- 3, TimeUnit.SECONDS);
- conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 5);
-
+ RatisClientConfig.RaftConfig raftClientConfig =
+ conf.getObject(RatisClientConfig.RaftConfig.class);
+ raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
+ raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3));
+ conf.setFromObject(raftClientConfig);
+
cluster = MiniOzoneCluster.newBuilder(conf)
.setHbProcessorInterval(1000)
.setHbInterval(1000)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org