You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/08/20 19:44:48 UTC
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into
HDDS-4440-s3-performance
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3b4e2f58b35f1e2dbf553ea183063867f1fd4771
Merge: cf8eb33 db574d1
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Fri Aug 20 07:36:19 2021 +0200
Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
.github/workflows/post-commit.yml | 130 ++-
SECURITY.md | 1 +
dev-support/ci/lib/_all_libs.sh | 34 +
dev-support/ci/lib/_initialization.sh | 140 +++
dev-support/ci/lib/_script_init.sh | 42 +
dev-support/ci/lib/_start_end.sh | 112 +++
dev-support/ci/lib/_traps.sh | 38 +
dev-support/ci/lib/_verbosity.sh | 57 ++
dev-support/ci/selective_ci_checks.bats | 210 ++++
dev-support/ci/selective_ci_checks.sh | 522 ++++++++++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 19 +
.../hadoop/hdds/client/ReplicationConfig.java | 10 +-
.../hdds/client/ReplicationConfigValidator.java | 63 ++
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 ------
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 +
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 +
.../hadoop/hdds/scm/net/NetworkTopology.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 +
.../protocol/StorageContainerLocationProtocol.java | 11 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../org/apache/hadoop/ozone/conf/package-info.java | 15 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../org/apache/hadoop/ozone/util/package-info.java | 15 +-
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 160 ++-
.../hadoop/hdds/client/TestReplicationConfig.java | 24 +
.../client/TestReplicationConfigValidator.java | 73 ++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +-
.../ozone/container/ContainerTestHelper.java | 75 ++
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 +
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 13 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 52 +-
.../common/statemachine/DatanodeStateMachine.java | 15 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 276 ++++--
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 8 +-
.../transport/server/ratis/XceiverServerRatis.java | 20 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db/package-info.java | 17 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../container/stream/DirectoryServerSource.java | 12 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 4 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../docs/content/feature/Nonrolling-Upgrade.md | 107 ++
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/_index.zh.md | 14 +-
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 --
hadoop-hdds/docs/content/interface/Ofs.md | 28 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
.../container/common/helpers/MoveDataNodePair.java | 71 ++
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 +
...inerLocationProtocolClientSideTranslatorPB.java | 30 +-
.../scm/update/client/CRLClientUpdateHandler.java | 5 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 2 +
.../hadoop/hdds/server/http/HttpServer2.java | 18 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
.../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 4 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 38 +-
.../interface-client/src/main/proto/hdds.proto | 5 +
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 122 ++-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 +++---
.../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ---------
.../hdds/scm/block/SCMBlockDeletingService.java | 18 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 186 ++++
.../scm/command/CommandStatusReportHandler.java | 6 +-
.../hdds/scm/container/ContainerReplicaCount.java | 1 +
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../hdds/scm/container/ReplicationManager.java | 953 +++++++++++++++---
.../scm/container/balancer/ContainerBalancer.java | 637 +++++++++---
.../balancer/ContainerBalancerConfiguration.java | 206 +++-
.../balancer/ContainerBalancerMetrics.java | 13 +-
.../ContainerBalancerSelectionCriteria.java | 169 ++++
.../container/balancer/ContainerMoveSelection.java | 55 ++
.../scm/container/balancer/FindTargetGreedy.java | 135 +++
.../scm/container/balancer/FindTargetStrategy.java | 71 ++
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 86 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../replication/ReplicationManagerMetrics.java | 161 ++++
.../scm/container/states/ContainerQueryKey.java | 33 +-
.../scm/container/states/ContainerStateMap.java | 41 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 10 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 +
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 9 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 +
...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 57 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +
.../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 801 ---------------
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 40 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 56 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hdds/scm/server/StorageContainerManager.java | 32 +-
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 7 +-
.../ScmHAUnfinalizedStateValidationAction.java | 13 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 67 +-
.../container/TestCloseContainerEventHandler.java | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 843 +++++++++++++---
.../scm/container/TestSCMContainerManager.java | 6 +-
.../container/balancer/TestContainerBalancer.java | 466 ++++++++-
.../algorithms/TestContainerPlacementFactory.java | 39 +-
.../TestSCMContainerPlacementCapacity.java | 2 +-
.../TestSCMContainerPlacementRackAware.java | 109 ++-
.../TestSCMContainerPlacementRandom.java | 70 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 -
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 14 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 ----------------
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 14 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 3 +
.../ozone/container/common/TestEndPoint.java | 3 +-
.../placement/TestContainerPlacement.java | 5 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
.../hdds/scm/cli/ContainerBalancerCommands.java | 19 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 16 +-
.../hdds/scm/cli/ContainerOperationClient.java | 26 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/container/ContainerCommands.java | 1 -
.../hdds/scm/cli/container/DeleteSubcommand.java | 54 --
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 61 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 54 +-
hadoop-ozone/common/pom.xml | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 55 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 -
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 52 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +-
.../main/resources/ozone-version-info.properties | 3 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 22 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/docker/Dockerfile | 38 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../{ozonesecure-mr => ozone-mr/hadoop33}/.env | 7 +-
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 +++
.../run.sh => ozone-mr/hadoop33/docker-config} | 11 +-
.../{ozone/run.sh => ozone-mr/hadoop33/test.sh} | 26 +-
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 +
.../src/main/compose/ozonesecure-ha/docker-config | 4 +
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 6 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 4 -
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../dist/src/main/smoketest/basic/links.robot | 10 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/boto3.robot | 5 +-
.../dist/src/main/smoketest/s3/boto_client.py | 10 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 6 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 3 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 10 +-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 39 +
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +-
.../ozone/contract/ITestOzoneContractDistCp.java | 8 +
.../rooted/ITestRootedOzoneContractDistCp.java | 8 +
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../TestContainerStateManagerIntegration.java | 40 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 ++++
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 5 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 235 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 5 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../ozone/TestContainerBalancerOperations.java | 11 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 12 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 130 ++-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 82 +-
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 369 ++++---
.../server/TestSecureContainerServer.java | 6 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +-
.../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +-
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 +++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 166 ++++
.../hadoop/ozone/scm/TestCloseContainer.java | 8 +
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +-
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 66 +-
.../src/main/proto/OmClientProtocol.proto | 10 +
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 416 ++++++--
.../hadoop/ozone/om/OzoneManagerStarter.java | 47 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 13 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 383 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 30 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 88 ++
.../om/request/bucket/OMBucketCreateRequest.java | 102 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 7 -
.../bucket/acl/OMBucketRemoveAclRequest.java | 7 -
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +-
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../request/s3/security/S3RevokeSecretRequest.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 76 +-
.../protocolPB/OzoneManagerRequestHandler.java | 8 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 6 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 9 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 1019 ++++++++++++++++++++
.../hadoop/ozone/recon/api/NodeEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/types/DUResponse.java | 177 ++++
.../hadoop/ozone/recon/api/types/EntityType.java | 16 +-
.../api/types/FileSizeDistributionResponse.java | 61 ++
.../hadoop/ozone/recon/api/types/NSSummary.java | 57 +-
.../recon/api/types/NamespaceSummaryResponse.java | 106 ++
.../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++
.../ozone/recon/api/types/ResponseStatus.java | 13 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 106 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../recon/scm/ReconPipelineReportHandler.java | 22 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 4 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 24 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +-
.../webapps/recon/ozone-recon-web/api/db.json | 324 ++++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 22 +-
.../src/components/navBar/navBar.tsx | 5 +
.../src/components/rightDrawer/rightDrawer.tsx | 72 ++
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 5 +
.../src/views/datanodes/datanodes.tsx | 8 +-
.../diskUsage/diskUsage.less} | 41 +-
.../src/views/diskUsage/diskUsage.tsx | 439 +++++++++
.../src/views/overview/overview.tsx | 3 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 109 +++
.../ozone/recon/api/TestNSSummaryEndpoint.java | 648 +++++++++++++
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../ozone/recon/scm/TestReconNodeManager.java | 38 +
.../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 17 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 40 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 6 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 20 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 52 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 19 +-
405 files changed, 17159 insertions(+), 5676 deletions(-)
diff --cc hadoop-ozone/interface-client/pom.xml
index 1695e4e,743dcb0..e6325ca
--- a/hadoop-ozone/interface-client/pom.xml
+++ b/hadoop-ozone/interface-client/pom.xml
@@@ -73,26 -55,74 +73,82 @@@ https://maven.apache.org/xsd/maven-4.0.
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
+ <goal>compile-custom</goal>
+ <goal>test-compile-custom</goal>
</goals>
<configuration>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
<protocArtifact>
- com.google.protobuf:protoc:${hadooprpc.protobuf.version}:exe:${os.detected.classifier}
+ com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}
</protocArtifact>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
- <includes>
- <include>OmClientProtocol.proto</include>
- <include>Security.proto</include>
- </includes>
+ <outputDirectory>target/generated-sources/protobuf/java</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>
+ io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}
+ </pluginArtifact>
</configuration>
</execution>
+ <execution>
+ <id>compile-protoc3</id>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ </goals>
+ <configuration>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+ <protocArtifact>
+ com.google.protobuf:protoc:${proto3.hadooprpc.protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+ <outputDirectory>target/generated-sources/protobuf/java/proto3</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <configuration>
+ <tasks>
+ <replace token="com.google.protobuf"
+ value="org.apache.hadoop.thirdparty.protobuf"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="com.google.protobuf"
+ value="org.apache.hadoop.thirdparty.protobuf"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.protocol.proto"
+ value="org.apache.hadoop.ozone.protocol.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.security.proto"
+ value="org.apache.hadoop.ozone.security.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto">
+ </replace>
+ <replace token="org.apache.hadoop.hdds.protocol.proto"
+ value="org.apache.hadoop.hdds.protocol.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.security.proto"
+ value="org.apache.hadoop.ozone.security.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
+ tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto3"/>
+ <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"
+ tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto3"/>
+ <move file="target/generated-sources/protobuf/java/proto3"
+ tofile="target/generated-sources/protobuf/java/"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
<plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org