You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2021/07/28 22:15:58 UTC
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into
HDDS-4944
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c826ec253297e24d8eddafeead71cc987ca32018
Merge: ab84d9e cd8287b
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Wed Jul 28 15:14:30 2021 -0700
Merge remote-tracking branch 'asf/master' into HDDS-4944
Conflicts:
hadoop-ozone/common/pom.xml
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
Change-Id: Iedf0713cfbba4ea95c27f98f74b6c6806b465acd
.github/workflows/post-commit.yml | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
hadoop-hdds/common/pom.xml | 12 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 9 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 17 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 76 ++
.../hdds/upgrade/HDDSLayoutVersionManager.java | 97 ++
.../hadoop/hdds/upgrade/HDDSUpgradeAction.java | 15 +-
.../apache/hadoop/hdds/upgrade}/package-info.java | 5 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../org/apache/hadoop/ozone/common/Storage.java | 41 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 33 +-
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../apache/hadoop/ozone/conf}/package-info.java | 12 +-
.../upgrade/AbstractLayoutVersionManager.java | 162 +++
.../ozone/upgrade/BasicUpgradeFinalizer.java | 367 +++++++
.../DefaultUpgradeFinalizationExecutor.java | 66 ++
.../apache/hadoop/ozone/upgrade/LayoutFeature.java | 79 ++
.../upgrade/LayoutVersionInstanceFactory.java | 253 +++++
.../hadoop/ozone/upgrade/LayoutVersionManager.java | 76 ++
.../ozone/upgrade/LayoutVersionManagerMXBean.java | 18 +-
.../hadoop/ozone/upgrade/UpgradeActionHdds.java | 39 +-
.../hadoop/ozone/upgrade/UpgradeException.java | 166 +--
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 195 ++++
.../hadoop/ozone/upgrade/VersionFactoryKey.java | 70 ++
.../apache/hadoop/ozone/upgrade}/package-info.java | 12 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../apache/hadoop/ozone/util}/package-info.java | 12 +-
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 9 +
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 102 ++
.../hadoop/hdds/upgrade/test/MockComponent.java | 67 ++
.../ozone/container/ContainerTestHelper.java | 75 ++
.../InjectedUpgradeFinalizationExecutor.java | 128 +++
.../upgrade/TestAbstractLayoutVersionManager.java | 203 ++++
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 126 +++
.../TestDefaultUpgradeFinalizationExecutor.java | 99 ++
.../upgrade/TestLayoutVersionInstanceFactory.java | 198 ++++
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 210 ++++
.../hadoop/ozone/upgrade/TestUpgradeUtils.java | 54 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 17 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../container/common/DatanodeLayoutStorage.java | 98 ++
...utVersion.java => HDDSVolumeLayoutVersion.java} | 16 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 53 +-
.../common/statemachine/StateContext.java | 3 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 2 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 121 +++
.../states/endpoint/HeartbeatEndpointTask.java | 65 +-
.../states/endpoint/RegisterEndpointTask.java | 60 +-
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 2 +-
.../container/common/utils/HddsVolumeUtil.java | 6 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db}/package-info.java | 10 +-
.../ozone/container/common/volume/HddsVolume.java | 7 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 4 +-
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 95 ++
.../upgrade/DatanodeMetadataFeatures.java | 53 +
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 48 +
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 +
.../UpgradeUtils.java} | 45 +-
.../ozone/container/upgrade}/package-info.java | 13 +-
.../protocol/StorageContainerDatanodeProtocol.java | 6 +-
.../protocol/StorageContainerNodeProtocol.java | 12 +-
.../commands/FinalizeNewLayoutVersionCommand.java | 80 ++
...inerDatanodeProtocolClientSideTranslatorPB.java | 11 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 16 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 11 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 12 +-
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 96 ++
.../docs/content/design/nonrolling-upgrade.md | 32 +
hadoop-hdds/docs/content/design/omprepare.md | 60 ++
.../docs/content/design/upgrade-dev-primer.md | 111 ++
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
...inerLocationProtocolClientSideTranslatorPB.java | 54 +-
.../scm/update/client/CRLClientUpdateHandler.java | 5 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 21 +
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 38 +-
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 23 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 73 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 549 ++++++++--
.../scm/container/balancer/ContainerBalancer.java | 4 +-
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 102 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 14 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 3 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 43 +-
...andler.java => HealthyReadOnlyNodeHandler.java} | 41 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 39 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 216 +++-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 14 +-
...va => ReadOnlyHealthyToHealthyNodeHandler.java} | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 170 +++-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 12 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 25 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 49 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 29 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 63 ++
.../hdds/scm/server/SCMClientProtocolServer.java | 38 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 31 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 24 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 9 +-
.../hdds/scm/server/StorageContainerManager.java | 113 ++-
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 163 +++
.../ScmHAUnfinalizedStateValidationAction.java | 72 ++
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 46 +
.../hdds/scm/server/upgrade}/package-info.java | 5 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 19 +
.../hadoop/hdds/scm/container/MockNodeManager.java | 26 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 40 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../hdds/scm/container/TestReplicationManager.java | 409 ++++++--
.../algorithms/TestContainerPlacementFactory.java | 41 +-
.../TestSCMContainerPlacementCapacity.java | 6 +-
.../TestSCMContainerPlacementRackAware.java | 112 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../hdds/scm/node/TestContainerPlacement.java | 23 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hdds/scm/node/TestNodeReportHandler.java | 16 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 57 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 532 +++++++++-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 17 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 26 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 10 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 128 +++
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++
.../hadoop/hdds/scm/upgrade/package-info.java | 21 +
.../ozone/container/common/TestEndPoint.java | 20 +-
.../placement/TestContainerPlacement.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 23 +-
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 63 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
.../hdds/scm/cli/ContainerOperationClient.java | 16 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../scm/cli/datanode/TestListInfoSubcommand.java | 13 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 21 +
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 82 ++
.../hadoop/ozone/om/exceptions/OMException.java | 7 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 +-
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 11 +
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 105 ++
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 38 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 111 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +-
.../main/resources/ozone-version-info.properties | 3 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 31 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 3 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
.../1.0.0/test.sh => ozone-mr/hadoop33/.env} | 13 +-
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 ++
.../test.sh => ozone-mr/hadoop33/docker-config} | 20 +-
.../compose/{upgrade => ozone-mr/hadoop33}/test.sh | 31 +-
.../compose/{upgrade => ozone-om-prepare}/.env | 10 +-
.../docker-compose.yaml | 75 +-
.../{upgrade => ozone-om-prepare}/docker-config | 28 +-
.../dist/src/main/compose/ozone-om-prepare/test.sh | 74 ++
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +-
.../dist/src/main/compose/upgrade/README.md | 95 +-
.../src/main/compose/upgrade/{ => compose/ha}/.env | 7 +-
.../upgrade/{ => compose/ha}/docker-compose.yaml | 120 ++-
.../compose/upgrade/{ => compose/ha}/docker-config | 30 +-
.../upgrade/{1.0.0/test.sh => compose/ha/load.sh} | 14 +-
.../main/compose/upgrade/{ => compose/non-ha}/.env | 2 +-
.../{ => compose/non-ha}/docker-compose.yaml | 99 +-
.../upgrade/{ => compose/non-ha}/docker-config | 13 +-
.../{0.5.0/test.sh => compose/non-ha/load.sh} | 13 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 30 +-
.../dist/src/main/compose/upgrade/testlib.sh | 135 ++-
.../src/main/compose/upgrade/upgrade_to_current.sh | 51 -
.../manual-upgrade/0.5.0-1.1.0/callback.sh} | 26 +-
.../manual-upgrade/driver.sh} | 63 +-
.../non-rolling-upgrade/1.0.0-1.1.0/callback.sh | 79 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 101 ++
.../dist/src/main/compose/versions/0.5.0.sh | 4 -
.../dist/src/main/compose/versions/1.0.0.sh | 4 -
.../dist/src/main/compose/versions/1.1.0.sh | 4 -
.../dist/src/main/compose/versions/README.md | 3 -
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 +
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 ++
.../LICENSE-org.reflections-reflections.txt | 14 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 69 +-
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../main/smoketest/omha/om-cancel-prepare.robot | 43 +
.../dist/src/main/smoketest/omha/om-prepare.robot | 49 +
.../bucketcreate.robot => omha/om-prepared.robot} | 29 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +-
.../bucketcreate.robot => upgrade/check-mlv.robot} | 30 +-
.../bucketcreate.robot => upgrade/finalize.robot} | 30 +-
.../bucketcreate.robot => upgrade/generate.robot} | 26 +-
.../bucketcreate.robot => upgrade/prepare.robot} | 25 +-
.../bucketcreate.robot => upgrade/validate.robot} | 23 +-
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} | 2 +-
.../upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh | 0
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 1067 ++++++++++++++++++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 25 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 52 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 263 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 7 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +
.../hadoop/ozone/TestStorageContainerManager.java | 29 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 81 +-
.../apache/hadoop/ozone/container/TestHelper.java | 6 +-
.../commandhandler/TestBlockDeletion.java | 266 +++--
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 254 +++++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 1 -
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 ++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 510 ++++++++++
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 163 +++
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 62 +-
.../src/main/proto/OmClientProtocol.proto | 95 +-
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 13 +
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/ozone-manager/pom.xml | 43 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 4 +
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 7 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 43 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 621 +++++++++---
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 311 ++++++
.../hadoop/ozone/om/OzoneManagerStarter.java | 79 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 15 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 405 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 87 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 97 ++
.../hadoop/ozone/om/request/OMClientRequest.java | 7 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 1 +
.../request/bucket/OMBucketSetPropertyRequest.java | 1 +
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 12 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../om/request/file/OMDirectoryCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 1 +
.../ozone/om/request/key/OMKeyDeleteRequest.java | 1 +
.../ozone/om/request/key/OMKeyPurgeRequest.java | 1 +
.../ozone/om/request/key/OMKeyRenameRequest.java | 1 +
.../om/request/key/acl/OMKeyAddAclRequest.java | 1 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 1 +
.../om/request/key/acl/OMKeySetAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixAddAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixSetAclRequest.java | 1 +
.../S3InitiateMultipartUploadRequest.java | 1 +
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../S3MultipartUploadCompleteRequest.java | 1 +
.../om/request/s3/security/S3GetSecretRequest.java | 1 +
.../security/OMCancelDelegationTokenRequest.java | 2 +-
.../security/OMGetDelegationTokenRequest.java | 2 +-
.../security/OMRenewDelegationTokenRequest.java | 1 +
.../om/request/upgrade/OMCancelPrepareRequest.java | 97 ++
.../request/upgrade/OMFinalizeUpgradeRequest.java | 116 +++
.../ozone/om/request/upgrade/OMPrepareRequest.java | 257 +++++
.../security => request/upgrade}/package-info.java | 15 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 1 +
.../om/request/volume/OMVolumeSetQuotaRequest.java | 1 +
.../hadoop/ozone/om/response/CleanupTableInfo.java | 10 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 1 +
.../om/response/s3/security/package-info.java | 2 +-
.../response/upgrade/OMCancelPrepareResponse.java | 46 +
.../upgrade/OMFinalizeUpgradeResponse.java | 58 ++
.../om/response/upgrade/OMPrepareResponse.java | 60 ++
.../{s3/security => upgrade}/package-info.java | 15 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../BelongsToLayoutVersion.java} | 23 +-
.../DisallowedUntilLayoutVersion.java} | 31 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 80 ++
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 114 +++
.../ozone/om/upgrade/OMLayoutVersionManager.java | 210 ++++
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 50 +
.../hadoop/ozone/om/upgrade/OmUpgradeAction.java | 19 +-
.../hadoop/ozone/om/upgrade/UpgradeActionOm.java | 37 +
.../s3/security => upgrade}/package-info.java | 7 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 97 +-
.../protocolPB/OzoneManagerRequestHandler.java | 70 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 9 +
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 34 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 108 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../upgrade/TestOMCancelPrepareRequest.java | 110 ++
.../request/volume/TestOMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 9 +-
.../ozone/om/response/TestCleanupTableInfo.java | 26 +-
.../hadoop/ozone/om/upgrade/MockOmRequest.java | 23 +-
.../ozone/om/upgrade/OMLayoutFeatureUtil.java | 66 ++
.../om/upgrade/TestOMLayoutFeatureAspect.java | 99 ++
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 290 ++++++
.../ozone/om/upgrade/TestOMVersionManager.java | 195 ++++
.../TestOmVersionManagerRequestFactory.java | 111 ++
.../om/upgrade/TestOzoneManagerPrepareState.java | 283 ++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
hadoop-ozone/recon/pom.xml | 5 +
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../ozone/recon/api/ClusterStateEndpoint.java | 3 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 781 ++++++++++++++
.../hadoop/ozone/recon/api/NodeEndpoint.java | 15 +-
.../hadoop/ozone/recon/api/types/DUResponse.java | 108 ++
.../ozone/recon/api/types/DatanodeMetadata.java | 13 +
.../hadoop/ozone/recon/api/types/EntityType.java} | 15 +-
.../api/types/FileSizeDistributionResponse.java | 61 ++
.../hadoop/ozone/recon/api/types/NSSummary.java | 57 +-
.../recon/api/types/NamespaceSummaryResponse.java | 105 ++
.../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++
.../ozone/recon/api/types/ResponseStatus.java | 13 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 139 ++-
.../scm/ReconStorageContainerManagerFacade.java | 10 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 4 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 24 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +-
.../src/views/datanodes/datanodes.tsx | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 86 ++
.../hadoop/ozone/recon/api/TestEndpoints.java | 13 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 498 +++++++++
.../ozone/recon/api/TestOpenContainerCount.java | 7 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../scm/AbstractReconContainerManagerTest.java | 13 +-
...TestReconIncrementalContainerReportHandler.java | 14 +-
.../ozone/recon/scm/TestReconNodeManager.java | 57 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 23 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 17 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++
hadoop-ozone/s3gateway/pom.xml | 5 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/admin/om/CancelPrepareSubCommand.java | 53 +
.../admin/om/FinalizationStatusSubCommand.java | 64 ++
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 171 ++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 44 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 173 ++++
.../admin/scm/FinalizationScmStatusSubcommand.java | 51 +
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 156 +++
.../admin/scm/FinalizeUpgradeCommandUtil.java | 92 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 4 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 15 +-
471 files changed, 22622 insertions(+), 2774 deletions(-)
diff --cc hadoop-ozone/common/pom.xml
index 873c6ef,67217c2..c994932
--- a/hadoop-ozone/common/pom.xml
+++ b/hadoop-ozone/common/pom.xml
@@@ -73,13 -73,9 +73,17 @@@ https://maven.apache.org/xsd/maven-4.0.
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-server-framework</artifactId>
+ </dependency>
++ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ </dependency>
</dependencies>
<build>
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 2efdb19,953aa23..149aaf0
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -237,12 -237,11 +237,19 @@@ public class OMException extends IOExce
DIRECTORY_NOT_EMPTY,
+ PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED,
+ REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED,
+ UPDATE_LAYOUT_VERSION_FAILED,
+ LAYOUT_FEATURE_FINALIZATION_FAILED,
+ PREPARE_FAILED,
- NOT_SUPPORTED_OPERATION_WHEN_PREPARED
++ NOT_SUPPORTED_OPERATION_WHEN_PREPARED,
++
+ TENANT_NOT_FOUND,
+ TENANT_ALREADY_EXISTS,
+ INVALID_TENANT_NAME,
+
+ TENANT_USER_NOT_FOUND,
+ TENANT_USER_ALREADY_EXISTS,
+ INVALID_TENANT_USER_NAME
}
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 1c3f9be,41604a3..9d741a4
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -357,13 -348,12 +373,20 @@@ enum Status
DIRECTORY_NOT_EMPTY = 68;
- TENANT_NOT_FOUND = 69;
- TENANT_ALREADY_EXISTS = 70;
- INVALID_TENANT_NAME = 71;
+ PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED = 69;
+ REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED = 70;
+ UPDATE_LAYOUT_VERSION_FAILED = 71;
+ LAYOUT_FEATURE_FINALIZATION_FAILED = 72;
+ PREPARE_FAILED = 73;
+ NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74;
+
- TENANT_USER_NOT_FOUND = 72;
- TENANT_USER_ALREADY_EXISTS = 73;
- INVALID_TENANT_USER_NAME = 74;
++ TENANT_NOT_FOUND = 75;
++ TENANT_ALREADY_EXISTS = 76;
++ INVALID_TENANT_NAME = 77;
++
++ TENANT_USER_NOT_FOUND = 78;
++ TENANT_USER_ALREADY_EXISTS = 79;
++ INVALID_TENANT_USER_NAME = 80;
}
/**
diff --cc hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 519bb08,dba75b4..760cc00
--- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@@ -350,21 -354,13 +357,27 @@@ public interface OMMetadataManager exte
Table<String, TransactionInfo> getTransactionInfoTable();
+ Table<String, String> getTenantUserTable();
+
+ Table<String, OmDBAccessIdInfo> getTenantAccessIdTable();
+
+ Table<String, OmDBKerberosPrincipalInfo> getPrincipalToAccessIdsTable();
+
+ Table<String, OmDBTenantInfo> getTenantStateTable();
+
+ Table<String, String> getTenantGroupTable();
+
+ Table<String, String> getTenantRoleTable();
+
+ Table<String, String> getTenantPolicyTable();
+
/**
+ * Gets the OM Meta table.
+ * @return meta table reference.
+ */
+ Table<String, String> getMetaTable();
+
+ /**
* Returns number of rows in a table. This should not be used for very
* large tables.
* @param table
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 8d8257b,a854ca4..6c81c93
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@@ -189,17 -171,27 +189,44 @@@ public class OmMetadataManagerImpl impl
public static final String DELETED_DIR_TABLE = "deletedDirectoryTable";
public static final String TRANSACTION_INFO_TABLE =
"transactionInfoTable";
+ public static final String META_TABLE = "metaTable";
+ // Tables for S3 multi-tenancy
+ public static final String TENANT_USER_TABLE = "tenantUserTable";
+ public static final String TENANT_ACCESS_ID_TABLE = "tenantAccessIdTable";
+ public static final String PRINCIPAL_TO_ACCESS_IDS_TABLE =
+ "principalToAccessIdsTable";
+ public static final String TENANT_STATE_TABLE = "tenantStateTable";
+ public static final String TENANT_GROUP_TABLE = "tenantGroupTable";
+ public static final String TENANT_ROLE_TABLE = "tenantRoleTable";
+ public static final String TENANT_POLICY_TABLE = "tenantPolicyTable";
+
+ static final String[] ALL_TABLES = new String[] {
+ USER_TABLE,
+ VOLUME_TABLE,
+ BUCKET_TABLE,
+ KEY_TABLE,
+ DELETED_TABLE,
+ OPEN_KEY_TABLE,
+ MULTIPARTINFO_TABLE,
+ S3_SECRET_TABLE,
+ DELEGATION_TOKEN_TABLE,
+ PREFIX_TABLE,
+ TRANSACTION_INFO_TABLE,
+ DIRECTORY_TABLE,
+ FILE_TABLE,
+ DELETED_DIR_TABLE,
+ OPEN_FILE_TABLE,
- META_TABLE
++ META_TABLE,
++ TENANT_USER_TABLE,
++ TENANT_ACCESS_ID_TABLE,
++ PRINCIPAL_TO_ACCESS_IDS_TABLE,
++ TENANT_STATE_TABLE,
++ TENANT_GROUP_TABLE,
++ TENANT_ROLE_TABLE,
++ TENANT_POLICY_TABLE
+ };
+
private DBStore store;
private final OzoneManagerLock lock;
@@@ -219,15 -211,7 +246,17 @@@
private Table fileTable;
private Table openFileTable;
private Table transactionInfoTable;
+ private Table metaTable;
++
+ // Tables for S3 multi-tenancy
+ private Table tenantUserTable;
+ private Table tenantAccessIdTable;
+ private Table principalToAccessIdsTable;
+ private Table tenantStateTable;
+ private Table tenantGroupTable;
+ private Table tenantRoleTable;
+ private Table tenantPolicyTable;
+
private boolean isRatisEnabled;
private boolean ignorePipelineinKey;
private Table deletedDirTable;
@@@ -426,13 -415,7 +460,14 @@@
.addTable(OPEN_FILE_TABLE)
.addTable(DELETED_DIR_TABLE)
.addTable(TRANSACTION_INFO_TABLE)
+ .addTable(META_TABLE)
+ .addTable(TENANT_USER_TABLE)
+ .addTable(TENANT_ACCESS_ID_TABLE)
+ .addTable(PRINCIPAL_TO_ACCESS_IDS_TABLE)
+ .addTable(TENANT_STATE_TABLE)
+ .addTable(TENANT_GROUP_TABLE)
+ .addTable(TENANT_ROLE_TABLE)
+ .addTable(TENANT_POLICY_TABLE)
.addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec())
.addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true))
.addCodec(RepeatedOmKeyInfo.class,
@@@ -519,42 -501,8 +554,45 @@@
String.class, TransactionInfo.class);
checkTableStatus(transactionInfoTable, TRANSACTION_INFO_TABLE);
+ metaTable = this.store.getTable(META_TABLE, String.class, String.class);
+ checkTableStatus(metaTable, META_TABLE);
++
+ // tenant user name -> tenant name string
+ tenantUserTable = this.store.getTable(TENANT_USER_TABLE,
+ String.class, String.class);
+ checkTableStatus(tenantUserTable, TENANT_USER_TABLE);
+
+ // tenantId -> OmDBAccessIdInfo (tenantId, secret, Kerberos principal)
+ tenantAccessIdTable = this.store.getTable(TENANT_ACCESS_ID_TABLE,
+ String.class, OmDBAccessIdInfo.class);
+ checkTableStatus(tenantAccessIdTable, TENANT_ACCESS_ID_TABLE);
+
+ // Kerberos principal -> OmDBKerberosPrincipalInfo (A list of accessIds)
+ principalToAccessIdsTable = this.store.getTable(
+ PRINCIPAL_TO_ACCESS_IDS_TABLE,
+ String.class, OmDBKerberosPrincipalInfo.class);
+ checkTableStatus(principalToAccessIdsTable, PRINCIPAL_TO_ACCESS_IDS_TABLE);
+
+ // tenant name -> tenant (tenant states)
+ tenantStateTable = this.store.getTable(TENANT_STATE_TABLE,
+ String.class, OmDBTenantInfo.class);
+ checkTableStatus(tenantStateTable, TENANT_STATE_TABLE);
+
+ // tenant user name -> list of tenant groups the user belongs to
+ tenantGroupTable = this.store.getTable(TENANT_GROUP_TABLE,
+ String.class, String.class /* TODO: Use custom list */);
+ checkTableStatus(tenantGroupTable, TENANT_GROUP_TABLE);
+
+ // tenant user name -> list of roles in a tenant. e.g. admin for "finance"
+ // TODO: Placeholder. Unused in the prototype.
+ tenantRoleTable = this.store.getTable(TENANT_ROLE_TABLE,
+ String.class, String.class /* TODO: Use custom list */);
+ checkTableStatus(tenantRoleTable, TENANT_ROLE_TABLE);
+
+ // tenant policy name -> list of tenant policies
+ tenantPolicyTable = this.store.getTable(TENANT_POLICY_TABLE,
+ String.class, String.class /* TODO: Use custom list */);
+ checkTableStatus(tenantPolicyTable, TENANT_POLICY_TABLE);
}
/**
@@@ -1290,41 -1238,10 +1328,46 @@@
}
@Override
+ public Table<String, String> getMetaTable() {
+ return metaTable;
+ }
+
++ @Override
+ public Table<String, String> getTenantUserTable() {
+ return tenantUserTable;
+ }
+
+ @Override
+ public Table<String, OmDBAccessIdInfo> getTenantAccessIdTable() {
+ return tenantAccessIdTable;
+ }
+
+ @Override
+ public Table<String, OmDBKerberosPrincipalInfo>
+ getPrincipalToAccessIdsTable() {
+ return principalToAccessIdsTable;
+ }
+
+ @Override
+ public Table<String, OmDBTenantInfo> getTenantStateTable() {
+ return tenantStateTable;
+ }
+
+ @Override
+ public Table<String, String> getTenantGroupTable() {
+ return tenantGroupTable;
+ }
+
+ @Override
+ public Table<String, String> getTenantRoleTable() {
+ return tenantRoleTable;
+ }
+
+ @Override
+ public Table<String, String> getTenantPolicyTable() {
+ return tenantPolicyTable;
+ }
+
/**
* Update store used by subclass.
*
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 36d605c,393632e..c5c6859
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -541,11 -574,9 +576,11 @@@ public final class OzoneManager extend
* When OM state is reloaded, these services are re-initialized with the
* new OM state.
*/
- private void instantiateServices() throws IOException {
+ private void instantiateServices(boolean withNewSnapshot) throws IOException {
metadataManager = new OmMetadataManagerImpl(configuration);
+ multiTenantManagr = new OMMultiTenantManagerImpl(metadataManager,
+ configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager, getKmsProvider(),
isRatisEnabled);
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index c1d53e6,4b2ac49..11694c6
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@@ -180,75 -177,14 +180,83 @@@ public class OMDBDefinition implements
String.class, new StringCodec(), OmKeyInfo.class,
new OmKeyInfoCodec(true));
+ public static final DBColumnFamilyDefinition<String, String>
+ META_TABLE = new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.META_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+
+ // Tables for S3 multi-tenancy
+
+ // TODO: this table will be removed with the disappearance of CreateUser API.
+ public static final DBColumnFamilyDefinition<String, String>
+ TENANT_USER_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_USER_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+
+ public static final DBColumnFamilyDefinition<String, OmDBAccessIdInfo>
+ TENANT_ACCESS_ID_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_ACCESS_ID_TABLE,
+ String.class, // accessId
+ new StringCodec(),
+ OmDBAccessIdInfo.class, // tenantId, secret, principal
+ new OmDBAccessIdInfoCodec());
+
+ public static final DBColumnFamilyDefinition<String,
+ OmDBKerberosPrincipalInfo>
+ PRINCIPAL_TO_ACCESS_IDS_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.PRINCIPAL_TO_ACCESS_IDS_TABLE,
+ String.class, // Kerberos principal
+ new StringCodec(),
+ OmDBKerberosPrincipalInfo.class, // List of accessIds
+ new OmDBKerberosPrincipalInfoCodec());
+
+ public static final DBColumnFamilyDefinition<String, OmDBTenantInfo>
+ TENANT_STATE_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_STATE_TABLE,
+ String.class,
+ new StringCodec(),
+ OmDBTenantInfo.class,
+ new OmDBTenantInfoCodec());
+
+ public static final DBColumnFamilyDefinition<String, String>
+ TENANT_GROUP_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_GROUP_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+
+ public static final DBColumnFamilyDefinition<String, String>
+ TENANT_ROLE_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_ROLE_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+
+ public static final DBColumnFamilyDefinition<String, String>
+ TENANT_POLICY_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_POLICY_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+
+ // End tables for S3 multi-tenancy
+
@Override
public String getName() {
return OzoneConsts.OM_DB_NAME;
@@@ -265,10 -201,7 +273,10 @@@
VOLUME_TABLE, OPEN_KEY_TABLE, KEY_TABLE,
BUCKET_TABLE, MULTIPART_INFO_TABLE, PREFIX_TABLE, DTOKEN_TABLE,
S3_SECRET_TABLE, TRANSACTION_INFO_TABLE, DIRECTORY_TABLE,
- FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE,
- FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE};
++ FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE,
+ TENANT_USER_TABLE, TENANT_ACCESS_ID_TABLE,
+ PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE, TENANT_GROUP_TABLE,
+ TENANT_ROLE_TABLE, TENANT_POLICY_TABLE };
}
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 28ca13f,e655b28..9a874c9
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@@ -228,20 -244,14 +250,26 @@@ public final class OzoneManagerRatisUti
return new S3GetSecretRequest(omRequest);
case RecoverTrash:
return new OMTrashRecoverRequest(omRequest);
+ case FinalizeUpgrade:
+ return new OMFinalizeUpgradeRequest(omRequest);
+ case Prepare:
+ return new OMPrepareRequest(omRequest);
+ case CancelPrepare:
+ return new OMCancelPrepareRequest(omRequest);
case RevokeS3Secret:
return new S3RevokeSecretRequest(omRequest);
+ case CreateTenant:
+ return new OMTenantCreateRequest(omRequest);
+ case ModifyTenant:
+ return new OMTenantModifyRequest(omRequest);
+ case DeleteTenant:
+ return new OMTenantDeleteRequest(omRequest);
+ case CreateTenantUser:
+ return new OMTenantUserCreateRequest(omRequest);
+ case ModifyTenantUser:
+ return new OMTenantUserModifyRequest(omRequest);
+ case DeleteTenantUser:
+ return new OMTenantUserDeleteRequest(omRequest);
default:
throw new IllegalStateException("Unrecognized write command " +
"type request" + cmdType);
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 247a9ca,7a25634..7983ae8
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@@ -132,27 -132,15 +133,22 @@@ public class OzoneManagerProtocolServer
if (OmUtils.isReadOnly(request)) {
return submitReadRequestToOM(request);
} else {
- raftServerStatus = omRatisServer.checkLeaderStatus();
- if (raftServerStatus == LEADER_AND_READY) {
- try {
- omClientRequest =
- OzoneManagerRatisUtils.createClientRequest(request);
- request = omClientRequest.preExecute(ozoneManager);
- } catch (IOException ex) {
- // As some of the preExecute returns error. So handle here.
- if (omClientRequest != null) {
- omClientRequest.handleRequestFailure(ozoneManager);
- }
- return createErrorResponse(request, ex);
- }
- OMResponse response = submitRequestToRatis(request);
- if (!response.getSuccess() && omClientRequest != null) {
+ checkLeaderStatus();
+ try {
- OMClientRequest omClientRequest = createClientRequest(request);
++ omClientRequest = createClientRequest(request);
+ request = omClientRequest.preExecute(ozoneManager);
+ } catch (IOException ex) {
+ // As some of the preExecute returns error. So handle here.
++ if (omClientRequest != null) {
+ omClientRequest.handleRequestFailure(ozoneManager);
+ }
- return response;
- } else {
- throw createLeaderErrorException(raftServerStatus);
+ return createErrorResponse(request, ex);
}
- return submitRequestToRatis(request);
++ OMResponse response = submitRequestToRatis(request);
++ if (!response.getSuccess() && omClientRequest != null) {
++ omClientRequest.handleRequestFailure(ozoneManager);
++ }
++ return response;
}
} else {
return submitRequestDirectlyToOM(request);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org