You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2022/08/08 14:46:24 UTC
[ozone] 01/01: Merge branch 'master' into HDDS-6517-Snapshot
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a0a9d221d022bd4ce396e03e186d71e8914cbe23
Merge: 2410551d2e 8e11723984
Author: Nandakumar <na...@apache.org>
AuthorDate: Mon Aug 8 20:15:27 2022 +0530
Merge branch 'master' into HDDS-6517-Snapshot
CONTRIBUTING.md | 146 ++--
.../annotations/ReplicateAnnotationProcessor.java | 77 ++
.../services/javax.annotation.processing.Processor | 3 +-
dev-support/ci/selective_ci_checks.bats | 12 +
dev-support/ci/selective_ci_checks.sh | 10 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 10 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 13 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 32 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 1 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hadoop/hdds/client/ReplicationConfig.java | 2 +-
.../hdds/client/ReplicationConfigValidator.java | 4 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 51 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +
.../protocol/StorageContainerLocationProtocol.java | 9 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 6 +
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../upgrade/AbstractLayoutVersionManager.java | 17 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 59 +-
.../DefaultUpgradeFinalizationExecutor.java | 6 +-
.../common/src/main/resources/ozone-default.xml | 8 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 17 +
.../ozone/container/ContainerTestHelper.java | 65 +-
.../InjectedUpgradeFinalizationExecutor.java | 3 -
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 105 ++-
.../hadoop/ozone/upgrade/UpgradeTestUtils.java | 59 ++
.../hadoop/hdds/conf/ConfigurationSource.java | 8 +
.../helpers/BlockDeletingServiceMetrics.java | 102 +++
.../ozone/container/common/impl/ContainerSet.java | 80 +-
.../container/common/impl/HddsDispatcher.java | 25 +-
.../common/statemachine/DatanodeConfiguration.java | 17 +
.../common/utils/db/DatanodeDBProfile.java | 26 +-
.../volume/CapacityVolumeChoosingPolicy.java | 6 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 6 +-
.../ozone/container/common/volume/VolumeInfo.java | 33 +-
.../ozone/container/common/volume/VolumeUsage.java | 9 +-
.../reconstruction/ECContainerOperationClient.java | 40 +
.../ECReconstructionCoordinator.java | 61 +-
.../ECReconstructionCoordinatorTask.java | 6 +
.../container/ec/reconstruction/TokenHelper.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 1 +
.../ozone/container/keyvalue/KeyValueHandler.java | 24 +-
.../container/keyvalue/helpers/ChunkUtils.java | 50 +-
.../background/BlockDeletingService.java | 33 +-
.../StaleRecoveringContainerScrubbingService.java | 93 +++
.../container/metadata/AbstractDatanodeStore.java | 12 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 9 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 5 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 15 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 83 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +
.../protocol/StorageContainerDatanodeProtocol.java | 3 +-
.../protocol/commands/DeleteContainerCommand.java | 20 +-
.../commands/ReconstructECContainersCommand.java | 15 +-
.../commands/ReplicateContainerCommand.java | 20 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 11 +
.../container/common/TestBlockDeletingService.java | 52 +-
.../common/TestKeyValueContainerData.java | 7 +
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 181 +++++
.../impl/TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 8 +-
.../common/volume/TestReservedVolumeSpace.java | 149 ++++
.../container/common/volume/TestVolumeSet.java | 4 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 22 +-
.../container/ozoneimpl/TestContainerReader.java | 26 +-
.../ReplicationSupervisorScheduling.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../TestReconstructionECContainersCommands.java | 16 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 69 ++
.../docs/content/feature/S3-Multi-Tenancy-Setup.md | 27 +-
hadoop-hdds/docs/content/feature/Topology.md | 4 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 5 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 13 +
.../certificate/authority/CertificateStore.java | 6 +-
.../certificate/authority/DefaultCAServer.java | 8 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 8 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 46 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 73 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 12 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 10 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 50 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 30 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 131 ++--
.../apache/hadoop/hdds/utils/db/TableConfig.java | 13 +-
.../hdds/utils/db/managed/ManagedBloomFilter.java | 27 +-
.../hdds/utils/db/managed/ManagedCheckpoint.java | 33 +
.../db/managed/ManagedColumnFamilyOptions.java | 48 ++
.../hdds/utils/db/managed/ManagedDBOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedEnvOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedFlushOptions.java | 27 +-
.../managed/ManagedIngestExternalFileOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedLRUCache.java | 30 +-
.../hdds/utils/db/managed/ManagedObject.java | 40 +-
.../hdds/utils/db/managed/ManagedOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedReadOptions.java | 30 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 81 ++
.../utils/db/managed/ManagedRocksIterator.java | 34 +
.../db/managed/ManagedRocksObjectMetrics.java | 60 ++
.../utils/db/managed/ManagedRocksObjectUtils.java | 52 ++
.../utils/db/managed/ManagedSstFileWriter.java | 33 +-
.../hdds/utils/db/managed/ManagedStatistics.java | 28 +-
.../db/managed/ManagedTransactionLogIterator.java | 30 +-
.../hdds/utils/db/managed/ManagedWriteBatch.java | 33 +-
.../hdds/utils/db/managed/ManagedWriteOptions.java | 30 +-
.../hadoop/hdds/utils/db/managed/package-info.java | 35 +
.../security/token/TestBlockTokenVerifier.java | 3 +-
.../security/token/TestContainerTokenVerifier.java | 3 +-
.../token/TestOzoneBlockTokenSecretManager.java | 51 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 8 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 18 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 15 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 13 +-
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 44 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 3 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 16 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 30 +-
.../scm/block/DeletedBlockLogStateManager.java | 11 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 33 +
.../container/AbstractContainerReportHandler.java | 10 +-
.../scm/container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 8 +-
.../hdds/scm/container/ContainerManagerImpl.java | 11 +-
.../hdds/scm/container/ContainerReplica.java | 4 +-
.../hdds/scm/container/ContainerReportHandler.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 7 +-
.../scm/container/ContainerStateManagerImpl.java | 42 +-
.../IncrementalContainerReportHandler.java | 4 +-
.../balancer/AbstractFindTargetGreedy.java | 14 +-
.../scm/container/balancer/ContainerBalancer.java | 65 +-
.../ContainerBalancerSelectionCriteria.java | 71 +-
.../FindTargetGreedyByNetworkTopology.java | 7 +-
.../balancer/FindTargetGreedyByUsageInfo.java | 6 +-
.../algorithms/SCMContainerPlacementCapacity.java | 7 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 +-
.../SCMContainerPlacementRackScatter.java | 16 +-
.../algorithms/SCMContainerPlacementRandom.java | 6 +-
.../replication/ContainerHealthResult.java | 12 +
.../replication/ECContainerReplicaCount.java | 3 +-
.../replication/ECOverReplicationHandler.java | 5 +
.../replication/ECUnderReplicationHandler.java | 97 ++-
.../replication/LegacyReplicationManager.java | 68 +-
.../replication/OverReplicatedProcessor.java | 114 +++
.../container/replication/ReplicationManager.java | 175 ++++-
.../replication/UnderReplicatedProcessor.java | 125 +++
.../replication/UnhealthyReplicationHandler.java | 9 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 8 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 1 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 32 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 3 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 3 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 3 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 3 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 7 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 3 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 29 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 50 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 13 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 14 +-
.../scm/pipeline/WritableContainerFactory.java | 3 +-
.../scm/pipeline/WritableContainerProvider.java | 4 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
.../pipeline/WritableRatisContainerProvider.java | 3 +-
...inerLocationProtocolServerSideTranslatorPB.java | 19 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 136 ++--
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 10 +-
.../hdds/scm/server/StorageContainerManager.java | 44 +-
.../scm/server/upgrade/FinalizationCheckpoint.java | 25 +-
.../scm/server/upgrade/FinalizationManager.java | 16 +
.../server/upgrade/FinalizationManagerImpl.java | 38 +
.../server/upgrade/FinalizationStateManager.java | 15 +-
.../upgrade/FinalizationStateManagerImpl.java | 150 +++-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 88 ++-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 18 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 58 +-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../scm/container/TestContainerManagerImpl.java | 7 +-
.../scm/container/TestContainerReportHandler.java | 25 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../container/balancer/TestContainerBalancer.java | 120 +--
.../container/balancer/TestFindTargetStrategy.java | 3 +-
.../TestSCMContainerPlacementCapacity.java | 10 +
.../TestSCMContainerPlacementRackAware.java | 5 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 96 ++-
.../replication/TestECOverReplicationHandler.java | 5 +
.../replication/TestECUnderReplicationHandler.java | 130 +++-
.../replication/TestLegacyReplicationManager.java | 95 ++-
.../replication/TestOverReplicatedProcessor.java | 124 +++
.../replication/TestReplicationManager.java | 78 +-
.../replication/TestUnderReplicatedProcessor.java | 179 +++++
.../hdds/scm/ha/TestBackgroundSCMService.java | 7 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 16 +-
.../TestPipelineDatanodesIntersection.java | 3 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 8 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 14 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 33 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 2 +-
.../pipeline/TestWritableECContainerProvider.java | 28 +-
.../hdds/scm/update/server/MockCRLStore.java | 4 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 6 +-
.../hdds/scm/upgrade/TestScmFinalization.java | 52 +-
.../org/apache/ozone/test/GenericTestUtils.java | 7 -
.../hdds/scm/cli/ContainerOperationClient.java | 5 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++---
.../apache/hadoop/ozone/client/OzoneClient.java | 13 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +-
.../checksum/AbstractBlockChecksumComputer.java | 5 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 82 +-
.../checksum/ReplicatedFileChecksumHelper.java | 16 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 32 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 30 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 19 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 72 +-
.../TestReplicatedBlockChecksumComputer.java | 56 +-
hadoop-ozone/common/pom.xml | 6 -
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 5 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 102 ++-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 240 ++++++
...vider.java => OMFailoverProxyProviderBase.java} | 304 ++------
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 +
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 7 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 18 +-
.../hadoop/ozone/om/multitenant/AccessPolicy.java | 2 +-
.../hadoop/ozone/om/multitenant/OzoneTenant.java | 10 +-
.../ozone/om/multitenant/RangerAccessPolicy.java | 4 +-
.../apache/hadoop/ozone/om/multitenant/Tenant.java | 12 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 12 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 27 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 20 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 10 +-
.../OMInterServiceProtocolClientSideImpl.java | 12 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 18 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 12 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 347 +++++++++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 508 ++++--------
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 42 +-
.../dist/src/main/compose/ozone/monitoring.conf | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 14 -
.../src/main/compose/ozonesecure/docker-config | 26 +-
.../ozonesecure/mockserverInitialization.json | 98 ---
.../main/compose/upgrade/compose/ha/docker-config | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 47 ++
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../smoketest/security/ozone-secure-tenant.robot | 4 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 13 +-
.../network-tests/src/test/blockade/README.md | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 54 ++
.../TestContainerStateManagerIntegration.java | 2 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 3 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 5 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 4 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 151 +++-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 230 +-----
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 272 +++++++
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 352 +++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 37 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 28 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 6 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 3 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 55 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 262 +++++++
.../commandhandler/TestBlockDeletion.java | 101 +--
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 17 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 6 +-
.../hadoop/ozone/om/OmFailoverProxyUtil.java | 4 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 302 ++++----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 36 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 6 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../ozone/om/multitenant/RangerUserRequest.java | 278 +++++++
...estMultiTenantAccessAuthorizerRangerPlugin.java | 269 -------
.../om/multitenant/TestMultiTenantVolume.java | 29 +
.../om/multitenant/TestRangerBGSyncService.java | 349 ++++-----
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 3 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 6 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 14 +
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 5 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 27 +-
.../TestResetDeletedBlockRetryCountShell.java | 169 ++++
.../src/main/proto/OmClientProtocol.proto | 11 +
hadoop-ozone/ozone-manager/pom.xml | 6 -
.../hadoop/ozone/om/OMMultiTenantManager.java | 112 ++-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 350 +++++----
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 58 +-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 20 +-
.../StringOMHashCodeGeneratorImpl.java | 16 +-
.../ozone/om/hashcodegenerator/package-info.java} | 17 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 27 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 3 +-
.../ozone/om/multitenant/CachedTenantState.java | 41 +
.../InMemoryMultiTenantAccessController.java | 49 +-
.../multitenant/MultiTenantAccessAuthorizer.java | 282 -------
.../MultiTenantAccessAuthorizerDummyPlugin.java | 194 -----
.../MultiTenantAccessAuthorizerRangerPlugin.java | 854 ---------------------
.../multitenant/MultiTenantAccessController.java | 190 +++--
.../om/multitenant/OMRangerBGSyncService.java | 194 +++--
.../RangerClientMultiTenantAccessController.java | 261 +++++--
.../RangerRestMultiTenantAccessController.java | 31 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 +-
.../metrics/OzoneManagerDoubleBufferMetrics.java | 13 +
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 7 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 7 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 8 +
.../request/s3/security/S3SecretRequestHelper.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 2 +
.../protocolPB/OzoneManagerRequestHandler.java | 15 +
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 32 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 126 ++-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 11 +-
.../TestMultiTenantAccessController.java | 97 ++-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +
.../bucket/TestOMBucketSetPropertyRequest.java | 8 +-
.../om/request/key/TestOMKeyCommitRequest.java | 43 ++
.../request/key/TestOMOpenKeysDeleteRequest.java | 71 +-
.../s3/security/TestS3GetSecretRequest.java | 11 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 8 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 17 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 1 +
.../ozone/recon/api/types/DatanodeMetadata.java | 14 +
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 11 +-
.../recon/scm/ReconPipelineReportHandler.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 18 +-
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 72 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 90 +--
.../ozone/recon/api/TestTaskStatusService.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 22 +-
.../recon/fsck/TestContainerHealthStatus.java | 12 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 16 +-
.../TestContainerHealthTaskRecordGenerator.java | 14 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 2 +
.../TestReconInternalSchemaDefinition.java | 21 +-
.../recon/persistence/TestSqlSchemaSetup.java | 6 +-
.../persistence/TestStatsSchemaDefinition.java | 29 +-
.../TestUtilizationSchemaDefinition.java | 40 +-
.../scm/AbstractReconContainerManagerTest.java | 27 +-
.../ozone/recon/scm/TestReconContainerManager.java | 42 +-
...TestReconIncrementalContainerReportHandler.java | 16 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +-
.../recon/scm/TestReconPipelineReportHandler.java | 6 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 21 +-
.../TestStorageContainerServiceProviderImpl.java | 12 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 10 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 9 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 44 +-
.../ozone/recon/tasks/TestTableCountTask.java | 10 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 35 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 131 ++--
.../ozone/s3/signature/AWSSignatureProcessor.java | 30 +-
.../hadoop/ozone/s3/signature/AuthOperation.java} | 36 +-
.../s3/signature/AuthorizationV2HeaderParser.java | 12 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 92 +--
.../s3/signature/AuthorizationV4QueryParser.java | 3 +-
.../hadoop/ozone/s3/signature/Credential.java | 14 +-
...Parser.java => MalformedResourceException.java} | 27 +-
.../hadoop/ozone/s3/signature/SignatureParser.java | 4 +-
.../apache/hadoop/ozone/s3/util/AuditUtils.java | 53 ++
.../hadoop/ozone/client/ClientProtocolStub.java | 573 ++++++++++++++
.../hadoop/ozone/client/ObjectStoreStub.java | 1 +
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/client/OzoneClientStub.java | 7 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 28 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 28 +-
.../signature/TestAuthorizationV2HeaderParser.java | 70 +-
.../signature/TestAuthorizationV4HeaderParser.java | 98 ++-
.../s3/signature/TestStringToSignProducer.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 5 +-
.../ozone/admin/om/UpdateRangerSubcommand.java | 96 +++
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 73 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 48 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 6 +-
.../ozone/debug/container/ContainerCommands.java | 6 +-
.../ozone/freon/ClosedContainerReplicator.java | 33 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 2 +-
.../ozone/freon/OmBucketReadWriteKeyOps.java | 14 +-
.../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 5 +-
pom.xml | 62 +-
457 files changed, 11878 insertions(+), 5997 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org