You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2022/08/08 14:46:23 UTC
[ozone] branch HDDS-6517-Snapshot updated (2410551d2e -> a0a9d221d0)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a change to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 2410551d2e HDDS-6849. OM changes for tracking Snapshot Metadata. (#3582)
add a02c6df497 HDDS-6900. Propagate TimeoutException for all SCM HA Ratis calls. (#3564)
add ab923a3538 HDDS-6794. EC: Analyze and add putBlock even on non writing node in the case of partial single stripe. (#3514)
add c48a7b09ae HDDS-6968. Addendum: [Multi-Tenant] Fix USER_MISMATCH error even on correct user. (#3578)
add d088d199bf HDDS-6982. EC: Attempt to cleanup the RECOVERING container when reconstruction failed at coordinator. (#3583)
add 7471d38f0a HDDS-6978. EC: Cleanup RECOVERING container on DN restarts (#3585)
add 5d87d0eb75 HDDS-6901. Configure HDDS volume reserved as percentage of the volume space. (#3532)
add 901a88debf HDDS-6991. Throw java.lang.NullPointerException when running setOwner (#3586)
add 77cd1691a5 HDDS-6725. Close Rocks objects properly in StorageContainerManager (#3549)
add f21e2a1e0f HDDS-6954. Migrate simple tests in ozone-recon to JUnit5 (#3556)
add e822caebe9 HDDS-6993. [Multi-Tenant] Set QuotaInBytes and QuotaInNamespace during Tenant Create. (#3590)
add a3db2dfc50 HDDS-6992. ListKeys : SubPaths with a mixture of files and dirs breaks sorted order (#3589)
add 2e8f5f30ea HDDS-6761. [SCM HA finalization] Handle restarts, crashes, and leader changes. (#3534)
add 27745b54dd HDDS-6889. EC: put key command with EC replication can use ReplicationConfig validator (#3565)
add aa3bbf9932 HDDS-6981. Add metrics for BlockDeletingService on Datanode (#3580)
add e1b6b76f0a HDDS-6989: EC: Add Test for RECOVERING container cleanup when failure. (#3587)
add 989ef1a9cf HDDS-6999. Correct volume parameter description for freon test tools (#3592)
add b7066604d3 HDDS-6990. AuthorizerLockImpl.java#tryWriteLockInOMRequest: move sanity args check to after tryWriteLockThrowOnTimeout(). (#3588)
add 6afe31a0c5 HDDS-6940. EC: Skip the EC container for balancer (#3547)
add 3f3577a452 HDDS-6909. [Multi-Tenant] Use RangerClient for Ranger operations (#3576)
add 843fac2fb6 HDDS-6433. Refactor OMFailoverProxyProvider to provide a base for OM and GrpcOM FailoverProxyProviders (#3389)
add e1f39bd506 HDDS-6998. Ignore keys with higher updateID in OpenKeyDeleteRequest (#3596)
add 912ff42bd1 HDDS-6994. Update Contributing guide (#3593)
add 59d8a96267 Add missing recon address to docker-config file. (#3597)
add af0c473586 HDDS-7005. Recursive delete through bucket link deletes linked data. (#3598)
add a66e43e283 HDDS-6987. EC: Implement RECOVERING Container Scrubber (#3591)
add 49f2f3b526 HDDS-7006. Incorrect usedNamespace caused by exceeding bytes quota (#3600)
add a09492cf8e HDDS-7012. Bump aws-java-sdk-s3 to 1.12.261 (#3602)
add 3a12bf811c HDDS-7007. Missing spaces in error messages (#3601)
add 9ebb4aa534 HDDS-6895. EC: ReplicationManager - Logic to process the under replicated queue and assign work to DNs (#3599)
add 2395d04fcb HDDS-6904. Cleanup customer related events in S3G logs (#3531)
add 0a019a4d95 HDDS-7020. EC: ReplicationManager - skip processing open containers (#3608)
add 5e17998eef HDDS-7009. Avoid calling getBucket in object APIs (#3594)
add 1e546103f0 HDDS-7016. EC: Implement the Over replication Processor (#3610)
add 0886f62e6c HDDS-6976. 0GB data moved by container balancer after successful iteration (#3604)
add 6ee6c35767 HDDS-6807. Remove tcnative.version property (#3605)
add 2795fda0c5 HDDS-6995. Update ranger-intg to v2.3.0 (#3603)
add 3c6a128ca2 HDDS-7015. Ensure excluded nodes have network location when used in placement policies (#3606)
add c57c9bca25 HDDS-7032. RDBStore#getUpdatesSince should throw SequenceNumberNotFou… (#3613)
add f5ab00c6bf HDDS-5251. Refactor ContainerTestHelper factory methods (#3612)
add c55c2dc474 HDDS-7024. Add queue size metric for OM double buffer (#3611)
add ca26bf00df HDDS-7034. File handle leak on datanode in PutBlock (#3616)
add c18295411d HDDS-6871. Bump netty to 4.1.79 (#3505)
add 8f16515fcc HDDS-7036. Enable RocksDB stats in monitoring sample config (#3619)
add 7de7da5056 HDDS-7021. [Multi-Tenant] Fix warning message when OM BG Sync is unable to get Ozone service policy version from Ranger (#3609)
add 58c3166094 HDDS-6767. Add network location in Recon's datanode page (#3429)
add 11cdb25fec HDDS-7041. Avoid ConcurrentModificationException in RepeatedOmKeyInfo (#3617)
add bea83f206f HDDS-7046. Leader count missing in Recon (#3621)
add d1cdf9994e HDDS-7042. Rebuilding tenant cache omits empty tenants (#3618)
add 72ca001178 HDDS-6980. support block composite-crc checksum (#3581)
add af059010b6 HDDS-6956. Initialize ozoneLockProvider earlier in OzoneManager (#3555)
add d80b6293e5 HDDS-6728. SCM UI not showing correct HA roles (#3401)
add 97c6832de9 HDDS-7057. EC: ReplicationManager - Over replication handler should set repIndex on delete cmds (#3628)
add 9f188e38b2 HDDS-7056. EC: Ensure replica index is maintained when replicating a container (#3627)
add c913760611 HDDS-7052. Topology Dynamic List document issue (#3624)
add 4393525367 HDDS-7055. NPE in ec.reconstruction.TokenHelper (#3630)
add 6b5dc0fa68 HDDS-6899. EC: remove warnings and errors from console during online reconstruction of data. (#3632)
add 0485fda4f7 HDDS-7050. Recon "/missing" endpoint return limit too high (#3622)
add 9e8c1900ba HDDS-7059. S3G should not call lookup twice for Get Object (#3633)
add daf2578256 HDDS-6533. support balancing EC container (#3455)
add 57d620dbd4 HDDS-7068: EC: Prematurely re-throwed the exception in reconstruction cleanup loop. (#3639)
add 511977ad67 HDDS-7048. EC: Add debug logging with exception info when stripe write failed (#3625)
add 2f03b16f87 HDDS-7069. EC: ReplicationManager - Track nodes already used when handing under replication (#3640)
add 334c6ee65b HDDS-7073. Incomplete audit log for KeyBlockToDelete (#3642)
add 3b89d4daac HDDS-5564. feature/PrefixFSO.md translation (#2582)
add ccf1b7c18f HDDS-6951. Replace bucket.listKeys() with bucket.listStatus() in OmBucketReadWriteKeyOps (#3554)
add 2396611e3c HDDS-7044. Ignore pr_title_check for selective checks (#3620)
add fa61f9e663 HDDS-7077. EC: Fix block deletion not allowed due to missing pipelineID (#3644)
add 15c9a5bca3 HDDS-7081. EC: ReplicationManager - UnderRep handler should handle duplicate indexes (#3649)
add 348fcb6157 HDDS-7051. Fix offset Condition in ECKeyOutputStream (#3623)
add ef218d8b68 HDDS-6662. Provide basic hash code generator for OBSKeyPathLockStrategy (#3358)
add 4459f47c11 HDDS-7063. Increment createContainer metric when container is created while writing chunk (#3643)
add 8f23e6bd54 HDDS-4970. Significant overhead when DataNode is over-subscribed (#3654)
add 51897d4aa1 HDDS-7023. Add CLI for reseting the expired deleted block txn (#3615)
add 192eff11a9 HDDS-7087. Manage RocksObjects to detect leaks (#3656)
add 8e11723984 HDDS-7085. Update gRPC to 1.48.1 to address OOM bug (#3659)
new a0a9d221d0 Merge branch 'master' into HDDS-6517-Snapshot
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CONTRIBUTING.md | 146 ++--
.../annotations/ReplicateAnnotationProcessor.java | 77 ++
.../services/javax.annotation.processing.Processor | 3 +-
dev-support/ci/selective_ci_checks.bats | 12 +
dev-support/ci/selective_ci_checks.sh | 10 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 10 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 13 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 32 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 1 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hadoop/hdds/client/ReplicationConfig.java | 2 +-
.../hdds/client/ReplicationConfigValidator.java | 4 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 51 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +
.../protocol/StorageContainerLocationProtocol.java | 9 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 6 +
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../upgrade/AbstractLayoutVersionManager.java | 17 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 59 +-
.../DefaultUpgradeFinalizationExecutor.java | 6 +-
.../common/src/main/resources/ozone-default.xml | 8 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 17 +
.../ozone/container/ContainerTestHelper.java | 65 +-
.../InjectedUpgradeFinalizationExecutor.java | 3 -
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 105 ++-
.../hadoop/ozone/upgrade/UpgradeTestUtils.java | 59 ++
.../hadoop/hdds/conf/ConfigurationSource.java | 8 +
.../helpers/BlockDeletingServiceMetrics.java | 102 +++
.../ozone/container/common/impl/ContainerSet.java | 80 +-
.../container/common/impl/HddsDispatcher.java | 25 +-
.../common/statemachine/DatanodeConfiguration.java | 17 +
.../common/utils/db/DatanodeDBProfile.java | 26 +-
.../volume/CapacityVolumeChoosingPolicy.java | 6 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 6 +-
.../ozone/container/common/volume/VolumeInfo.java | 33 +-
.../ozone/container/common/volume/VolumeUsage.java | 9 +-
.../reconstruction/ECContainerOperationClient.java | 40 +
.../ECReconstructionCoordinator.java | 61 +-
.../ECReconstructionCoordinatorTask.java | 6 +
.../container/ec/reconstruction/TokenHelper.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 1 +
.../ozone/container/keyvalue/KeyValueHandler.java | 24 +-
.../container/keyvalue/helpers/ChunkUtils.java | 50 +-
.../background/BlockDeletingService.java | 33 +-
.../StaleRecoveringContainerScrubbingService.java | 93 +++
.../container/metadata/AbstractDatanodeStore.java | 12 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 9 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 5 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 15 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 83 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +
.../protocol/StorageContainerDatanodeProtocol.java | 3 +-
.../protocol/commands/DeleteContainerCommand.java | 20 +-
.../commands/ReconstructECContainersCommand.java | 15 +-
.../commands/ReplicateContainerCommand.java | 20 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 11 +
.../container/common/TestBlockDeletingService.java | 52 +-
.../common/TestKeyValueContainerData.java | 7 +
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 181 +++++
.../impl/TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 8 +-
.../common/volume/TestReservedVolumeSpace.java | 149 ++++
.../container/common/volume/TestVolumeSet.java | 4 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 22 +-
.../container/ozoneimpl/TestContainerReader.java | 26 +-
.../ReplicationSupervisorScheduling.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../TestReconstructionECContainersCommands.java | 16 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 69 ++
.../docs/content/feature/S3-Multi-Tenancy-Setup.md | 27 +-
hadoop-hdds/docs/content/feature/Topology.md | 4 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 5 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 13 +
.../certificate/authority/CertificateStore.java | 6 +-
.../certificate/authority/DefaultCAServer.java | 8 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 8 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 46 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 73 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 12 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 10 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 50 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 30 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 131 ++--
.../apache/hadoop/hdds/utils/db/TableConfig.java | 13 +-
.../hdds/utils/db/managed/ManagedBloomFilter.java | 27 +-
.../hdds/utils/db/managed/ManagedCheckpoint.java | 33 +
.../db/managed/ManagedColumnFamilyOptions.java | 48 ++
.../hdds/utils/db/managed/ManagedDBOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedEnvOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedFlushOptions.java | 27 +-
.../managed/ManagedIngestExternalFileOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedLRUCache.java | 30 +-
.../hdds/utils/db/managed/ManagedObject.java | 40 +-
.../hdds/utils/db/managed/ManagedOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedReadOptions.java | 30 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 81 ++
.../utils/db/managed/ManagedRocksIterator.java | 34 +
.../db/managed/ManagedRocksObjectMetrics.java | 60 ++
.../utils/db/managed/ManagedRocksObjectUtils.java | 52 ++
.../utils/db/managed/ManagedSstFileWriter.java | 33 +-
.../hdds/utils/db/managed/ManagedStatistics.java | 28 +-
.../db/managed/ManagedTransactionLogIterator.java | 30 +-
.../hdds/utils/db/managed/ManagedWriteBatch.java | 33 +-
.../hdds/utils/db/managed/ManagedWriteOptions.java | 30 +-
.../hadoop/hdds/utils/db/managed/package-info.java | 35 +
.../security/token/TestBlockTokenVerifier.java | 3 +-
.../security/token/TestContainerTokenVerifier.java | 3 +-
.../token/TestOzoneBlockTokenSecretManager.java | 51 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 8 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 18 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 15 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 13 +-
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 44 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 3 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 16 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 30 +-
.../scm/block/DeletedBlockLogStateManager.java | 11 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 33 +
.../container/AbstractContainerReportHandler.java | 10 +-
.../scm/container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 8 +-
.../hdds/scm/container/ContainerManagerImpl.java | 11 +-
.../hdds/scm/container/ContainerReplica.java | 4 +-
.../hdds/scm/container/ContainerReportHandler.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 7 +-
.../scm/container/ContainerStateManagerImpl.java | 42 +-
.../IncrementalContainerReportHandler.java | 4 +-
.../balancer/AbstractFindTargetGreedy.java | 14 +-
.../scm/container/balancer/ContainerBalancer.java | 65 +-
.../ContainerBalancerSelectionCriteria.java | 71 +-
.../FindTargetGreedyByNetworkTopology.java | 7 +-
.../balancer/FindTargetGreedyByUsageInfo.java | 6 +-
.../algorithms/SCMContainerPlacementCapacity.java | 7 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 +-
.../SCMContainerPlacementRackScatter.java | 16 +-
.../algorithms/SCMContainerPlacementRandom.java | 6 +-
.../replication/ContainerHealthResult.java | 12 +
.../replication/ECContainerReplicaCount.java | 3 +-
.../replication/ECOverReplicationHandler.java | 5 +
.../replication/ECUnderReplicationHandler.java | 97 ++-
.../replication/LegacyReplicationManager.java | 68 +-
.../replication/OverReplicatedProcessor.java | 114 +++
.../container/replication/ReplicationManager.java | 175 ++++-
.../replication/UnderReplicatedProcessor.java | 125 +++
.../replication/UnhealthyReplicationHandler.java | 9 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 8 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 1 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 32 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 3 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 3 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 3 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 3 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 7 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 3 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 29 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 50 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 13 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 14 +-
.../scm/pipeline/WritableContainerFactory.java | 3 +-
.../scm/pipeline/WritableContainerProvider.java | 4 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
.../pipeline/WritableRatisContainerProvider.java | 3 +-
...inerLocationProtocolServerSideTranslatorPB.java | 19 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 136 ++--
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 10 +-
.../hdds/scm/server/StorageContainerManager.java | 44 +-
.../scm/server/upgrade/FinalizationCheckpoint.java | 25 +-
.../scm/server/upgrade/FinalizationManager.java | 16 +
.../server/upgrade/FinalizationManagerImpl.java | 38 +
.../server/upgrade/FinalizationStateManager.java | 15 +-
.../upgrade/FinalizationStateManagerImpl.java | 150 +++-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 88 ++-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 18 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 58 +-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../scm/container/TestContainerManagerImpl.java | 7 +-
.../scm/container/TestContainerReportHandler.java | 25 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../container/balancer/TestContainerBalancer.java | 120 +--
.../container/balancer/TestFindTargetStrategy.java | 3 +-
.../TestSCMContainerPlacementCapacity.java | 10 +
.../TestSCMContainerPlacementRackAware.java | 5 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 96 ++-
.../replication/TestECOverReplicationHandler.java | 5 +
.../replication/TestECUnderReplicationHandler.java | 130 +++-
.../replication/TestLegacyReplicationManager.java | 95 ++-
.../replication/TestOverReplicatedProcessor.java | 124 +++
.../replication/TestReplicationManager.java | 78 +-
.../replication/TestUnderReplicatedProcessor.java | 179 +++++
.../hdds/scm/ha/TestBackgroundSCMService.java | 7 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 16 +-
.../TestPipelineDatanodesIntersection.java | 3 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 8 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 14 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 33 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 2 +-
.../pipeline/TestWritableECContainerProvider.java | 28 +-
.../hdds/scm/update/server/MockCRLStore.java | 4 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 6 +-
.../hdds/scm/upgrade/TestScmFinalization.java | 52 +-
.../org/apache/ozone/test/GenericTestUtils.java | 7 -
.../hdds/scm/cli/ContainerOperationClient.java | 5 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++---
.../apache/hadoop/ozone/client/OzoneClient.java | 13 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +-
.../checksum/AbstractBlockChecksumComputer.java | 5 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 82 +-
.../checksum/ReplicatedFileChecksumHelper.java | 16 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 32 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 30 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 19 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 72 +-
.../TestReplicatedBlockChecksumComputer.java | 56 +-
hadoop-ozone/common/pom.xml | 6 -
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 5 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 102 ++-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 240 ++++++
...vider.java => OMFailoverProxyProviderBase.java} | 304 ++------
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 +
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 7 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 18 +-
.../hadoop/ozone/om/multitenant/AccessPolicy.java | 2 +-
.../hadoop/ozone/om/multitenant/OzoneTenant.java | 10 +-
.../ozone/om/multitenant/RangerAccessPolicy.java | 4 +-
.../apache/hadoop/ozone/om/multitenant/Tenant.java | 12 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 12 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 27 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 20 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 10 +-
.../OMInterServiceProtocolClientSideImpl.java | 12 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 18 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 12 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 347 +++++++++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 508 ++++--------
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 42 +-
.../dist/src/main/compose/ozone/monitoring.conf | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 14 -
.../src/main/compose/ozonesecure/docker-config | 26 +-
.../ozonesecure/mockserverInitialization.json | 98 ---
.../main/compose/upgrade/compose/ha/docker-config | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 47 ++
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../smoketest/security/ozone-secure-tenant.robot | 4 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 13 +-
.../network-tests/src/test/blockade/README.md | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 54 ++
.../TestContainerStateManagerIntegration.java | 2 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 3 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 5 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 4 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 151 +++-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 230 +-----
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 272 +++++++
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 352 +++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 37 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 28 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 6 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 3 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 55 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 262 +++++++
.../commandhandler/TestBlockDeletion.java | 101 +--
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 17 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 6 +-
.../hadoop/ozone/om/OmFailoverProxyUtil.java | 4 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 302 ++++----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 36 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 6 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../ozone/om/multitenant/RangerUserRequest.java | 278 +++++++
...estMultiTenantAccessAuthorizerRangerPlugin.java | 269 -------
.../om/multitenant/TestMultiTenantVolume.java | 29 +
.../om/multitenant/TestRangerBGSyncService.java | 349 ++++-----
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 3 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 6 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 14 +
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 5 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 27 +-
.../TestResetDeletedBlockRetryCountShell.java | 169 ++++
.../src/main/proto/OmClientProtocol.proto | 11 +
hadoop-ozone/ozone-manager/pom.xml | 6 -
.../hadoop/ozone/om/OMMultiTenantManager.java | 112 ++-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 350 +++++----
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 58 +-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 20 +-
.../StringOMHashCodeGeneratorImpl.java | 16 +-
.../ozone/om/hashcodegenerator/package-info.java} | 17 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 27 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 3 +-
.../ozone/om/multitenant/CachedTenantState.java | 41 +
.../InMemoryMultiTenantAccessController.java | 49 +-
.../multitenant/MultiTenantAccessAuthorizer.java | 282 -------
.../MultiTenantAccessAuthorizerDummyPlugin.java | 194 -----
.../MultiTenantAccessAuthorizerRangerPlugin.java | 854 ---------------------
.../multitenant/MultiTenantAccessController.java | 190 +++--
.../om/multitenant/OMRangerBGSyncService.java | 194 +++--
.../RangerClientMultiTenantAccessController.java | 261 +++++--
.../RangerRestMultiTenantAccessController.java | 31 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 +-
.../metrics/OzoneManagerDoubleBufferMetrics.java | 13 +
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 7 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 7 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 8 +
.../request/s3/security/S3SecretRequestHelper.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 2 +
.../protocolPB/OzoneManagerRequestHandler.java | 15 +
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 32 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 126 ++-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 11 +-
.../TestMultiTenantAccessController.java | 97 ++-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +
.../bucket/TestOMBucketSetPropertyRequest.java | 8 +-
.../om/request/key/TestOMKeyCommitRequest.java | 43 ++
.../request/key/TestOMOpenKeysDeleteRequest.java | 71 +-
.../s3/security/TestS3GetSecretRequest.java | 11 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 8 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 17 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 1 +
.../ozone/recon/api/types/DatanodeMetadata.java | 14 +
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 11 +-
.../recon/scm/ReconPipelineReportHandler.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 18 +-
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 72 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 90 +--
.../ozone/recon/api/TestTaskStatusService.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 22 +-
.../recon/fsck/TestContainerHealthStatus.java | 12 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 16 +-
.../TestContainerHealthTaskRecordGenerator.java | 14 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 2 +
.../TestReconInternalSchemaDefinition.java | 21 +-
.../recon/persistence/TestSqlSchemaSetup.java | 6 +-
.../persistence/TestStatsSchemaDefinition.java | 29 +-
.../TestUtilizationSchemaDefinition.java | 40 +-
.../scm/AbstractReconContainerManagerTest.java | 27 +-
.../ozone/recon/scm/TestReconContainerManager.java | 42 +-
...TestReconIncrementalContainerReportHandler.java | 16 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +-
.../recon/scm/TestReconPipelineReportHandler.java | 6 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 21 +-
.../TestStorageContainerServiceProviderImpl.java | 12 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 10 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 9 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 44 +-
.../ozone/recon/tasks/TestTableCountTask.java | 10 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 35 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 131 ++--
.../ozone/s3/signature/AWSSignatureProcessor.java | 30 +-
.../hadoop/ozone/s3/signature/AuthOperation.java} | 36 +-
.../s3/signature/AuthorizationV2HeaderParser.java | 12 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 92 +--
.../s3/signature/AuthorizationV4QueryParser.java | 3 +-
.../hadoop/ozone/s3/signature/Credential.java | 14 +-
...Parser.java => MalformedResourceException.java} | 27 +-
.../hadoop/ozone/s3/signature/SignatureParser.java | 4 +-
.../apache/hadoop/ozone/s3/util/AuditUtils.java | 53 ++
.../hadoop/ozone/client/ClientProtocolStub.java | 573 ++++++++++++++
.../hadoop/ozone/client/ObjectStoreStub.java | 1 +
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/client/OzoneClientStub.java | 7 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 28 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 28 +-
.../signature/TestAuthorizationV2HeaderParser.java | 70 +-
.../signature/TestAuthorizationV4HeaderParser.java | 98 ++-
.../s3/signature/TestStringToSignProducer.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 5 +-
.../ozone/admin/om/UpdateRangerSubcommand.java | 96 +++
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 73 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 48 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 6 +-
.../ozone/debug/container/ContainerCommands.java | 6 +-
.../ozone/freon/ClosedContainerReplicator.java | 33 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 2 +-
.../ozone/freon/OmBucketReadWriteKeyOps.java | 14 +-
.../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 5 +-
pom.xml | 62 +-
457 files changed, 11878 insertions(+), 5997 deletions(-)
create mode 100644 dev-support/annotations/src/main/java/org/apache/ozone/annotations/ReplicateAnnotationProcessor.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
create mode 100644 hadoop-hdds/docs/content/feature/PrefixFSO.zh.md
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java (61%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java (61%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java (61%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java (61%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java (59%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java (59%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java (50%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java (61%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java (59%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java (52%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java (61%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java (53%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java (57%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java (59%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/OverReplicatedProcessor.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnderReplicatedProcessor.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestOverReplicatedProcessor.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestUnderReplicatedProcessor.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/HadoopRpcOMFailoverProxyProvider.java
rename hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/{OMFailoverProxyProvider.java => OMFailoverProxyProviderBase.java} (65%)
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure/mockserverInitialization.json
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/RangerUserRequest.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantAccessAuthorizerRangerPlugin.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestResetDeletedBlockRetryCountShell.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator/OMHashCodeGenerator.java (70%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator/StringOMHashCodeGeneratorImpl.java (70%)
copy hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java => ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator/package-info.java} (66%)
rename hadoop-ozone/ozone-manager/src/{test => main}/java/org/apache/hadoop/ozone/om/multitenant/InMemoryMultiTenantAccessController.java (72%)
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizer.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizerDummyPlugin.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizerRangerPlugin.java
copy hadoop-ozone/{client/src/main/java/org/apache/hadoop/ozone/client/checksum/AbstractBlockChecksumComputer.java => s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthOperation.java} (53%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/{SignatureParser.java => MalformedResourceException.java} (61%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/AuditUtils.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/UpdateRangerSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ResetDeletedBlockRetryCountSubcommand.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge branch 'master' into HDDS-6517-Snapshot
Posted by na...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a0a9d221d022bd4ce396e03e186d71e8914cbe23
Merge: 2410551d2e 8e11723984
Author: Nandakumar <na...@apache.org>
AuthorDate: Mon Aug 8 20:15:27 2022 +0530
Merge branch 'master' into HDDS-6517-Snapshot
CONTRIBUTING.md | 146 ++--
.../annotations/ReplicateAnnotationProcessor.java | 77 ++
.../services/javax.annotation.processing.Processor | 3 +-
dev-support/ci/selective_ci_checks.bats | 12 +
dev-support/ci/selective_ci_checks.sh | 10 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 10 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 13 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 32 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 1 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hadoop/hdds/client/ReplicationConfig.java | 2 +-
.../hdds/client/ReplicationConfigValidator.java | 4 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 51 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +
.../protocol/StorageContainerLocationProtocol.java | 9 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 6 +
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../upgrade/AbstractLayoutVersionManager.java | 17 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 59 +-
.../DefaultUpgradeFinalizationExecutor.java | 6 +-
.../common/src/main/resources/ozone-default.xml | 8 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 17 +
.../ozone/container/ContainerTestHelper.java | 65 +-
.../InjectedUpgradeFinalizationExecutor.java | 3 -
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 105 ++-
.../hadoop/ozone/upgrade/UpgradeTestUtils.java | 59 ++
.../hadoop/hdds/conf/ConfigurationSource.java | 8 +
.../helpers/BlockDeletingServiceMetrics.java | 102 +++
.../ozone/container/common/impl/ContainerSet.java | 80 +-
.../container/common/impl/HddsDispatcher.java | 25 +-
.../common/statemachine/DatanodeConfiguration.java | 17 +
.../common/utils/db/DatanodeDBProfile.java | 26 +-
.../volume/CapacityVolumeChoosingPolicy.java | 6 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 6 +-
.../ozone/container/common/volume/VolumeInfo.java | 33 +-
.../ozone/container/common/volume/VolumeUsage.java | 9 +-
.../reconstruction/ECContainerOperationClient.java | 40 +
.../ECReconstructionCoordinator.java | 61 +-
.../ECReconstructionCoordinatorTask.java | 6 +
.../container/ec/reconstruction/TokenHelper.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 1 +
.../ozone/container/keyvalue/KeyValueHandler.java | 24 +-
.../container/keyvalue/helpers/ChunkUtils.java | 50 +-
.../background/BlockDeletingService.java | 33 +-
.../StaleRecoveringContainerScrubbingService.java | 93 +++
.../container/metadata/AbstractDatanodeStore.java | 12 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 9 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 5 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 15 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 83 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +
.../protocol/StorageContainerDatanodeProtocol.java | 3 +-
.../protocol/commands/DeleteContainerCommand.java | 20 +-
.../commands/ReconstructECContainersCommand.java | 15 +-
.../commands/ReplicateContainerCommand.java | 20 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 11 +
.../container/common/TestBlockDeletingService.java | 52 +-
.../common/TestKeyValueContainerData.java | 7 +
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 181 +++++
.../impl/TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 8 +-
.../common/volume/TestReservedVolumeSpace.java | 149 ++++
.../container/common/volume/TestVolumeSet.java | 4 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 22 +-
.../container/ozoneimpl/TestContainerReader.java | 26 +-
.../ReplicationSupervisorScheduling.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../TestReconstructionECContainersCommands.java | 16 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 69 ++
.../docs/content/feature/S3-Multi-Tenancy-Setup.md | 27 +-
hadoop-hdds/docs/content/feature/Topology.md | 4 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 5 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 13 +
.../certificate/authority/CertificateStore.java | 6 +-
.../certificate/authority/DefaultCAServer.java | 8 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 8 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 46 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 73 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 12 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 10 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 50 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 30 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 131 ++--
.../apache/hadoop/hdds/utils/db/TableConfig.java | 13 +-
.../hdds/utils/db/managed/ManagedBloomFilter.java | 27 +-
.../hdds/utils/db/managed/ManagedCheckpoint.java | 33 +
.../db/managed/ManagedColumnFamilyOptions.java | 48 ++
.../hdds/utils/db/managed/ManagedDBOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedEnvOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedFlushOptions.java | 27 +-
.../managed/ManagedIngestExternalFileOptions.java | 28 +-
.../hdds/utils/db/managed/ManagedLRUCache.java | 30 +-
.../hdds/utils/db/managed/ManagedObject.java | 40 +-
.../hdds/utils/db/managed/ManagedOptions.java | 27 +-
.../hdds/utils/db/managed/ManagedReadOptions.java | 30 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 81 ++
.../utils/db/managed/ManagedRocksIterator.java | 34 +
.../db/managed/ManagedRocksObjectMetrics.java | 60 ++
.../utils/db/managed/ManagedRocksObjectUtils.java | 52 ++
.../utils/db/managed/ManagedSstFileWriter.java | 33 +-
.../hdds/utils/db/managed/ManagedStatistics.java | 28 +-
.../db/managed/ManagedTransactionLogIterator.java | 30 +-
.../hdds/utils/db/managed/ManagedWriteBatch.java | 33 +-
.../hdds/utils/db/managed/ManagedWriteOptions.java | 30 +-
.../hadoop/hdds/utils/db/managed/package-info.java | 35 +
.../security/token/TestBlockTokenVerifier.java | 3 +-
.../security/token/TestContainerTokenVerifier.java | 3 +-
.../token/TestOzoneBlockTokenSecretManager.java | 51 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 8 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 18 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 15 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 13 +-
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 44 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 3 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 16 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 30 +-
.../scm/block/DeletedBlockLogStateManager.java | 11 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 33 +
.../container/AbstractContainerReportHandler.java | 10 +-
.../scm/container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 8 +-
.../hdds/scm/container/ContainerManagerImpl.java | 11 +-
.../hdds/scm/container/ContainerReplica.java | 4 +-
.../hdds/scm/container/ContainerReportHandler.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 7 +-
.../scm/container/ContainerStateManagerImpl.java | 42 +-
.../IncrementalContainerReportHandler.java | 4 +-
.../balancer/AbstractFindTargetGreedy.java | 14 +-
.../scm/container/balancer/ContainerBalancer.java | 65 +-
.../ContainerBalancerSelectionCriteria.java | 71 +-
.../FindTargetGreedyByNetworkTopology.java | 7 +-
.../balancer/FindTargetGreedyByUsageInfo.java | 6 +-
.../algorithms/SCMContainerPlacementCapacity.java | 7 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 +-
.../SCMContainerPlacementRackScatter.java | 16 +-
.../algorithms/SCMContainerPlacementRandom.java | 6 +-
.../replication/ContainerHealthResult.java | 12 +
.../replication/ECContainerReplicaCount.java | 3 +-
.../replication/ECOverReplicationHandler.java | 5 +
.../replication/ECUnderReplicationHandler.java | 97 ++-
.../replication/LegacyReplicationManager.java | 68 +-
.../replication/OverReplicatedProcessor.java | 114 +++
.../container/replication/ReplicationManager.java | 175 ++++-
.../replication/UnderReplicatedProcessor.java | 125 +++
.../replication/UnhealthyReplicationHandler.java | 9 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 8 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 1 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 32 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 3 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 3 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 3 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 3 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 7 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 3 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 29 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 50 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 13 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 14 +-
.../scm/pipeline/WritableContainerFactory.java | 3 +-
.../scm/pipeline/WritableContainerProvider.java | 4 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
.../pipeline/WritableRatisContainerProvider.java | 3 +-
...inerLocationProtocolServerSideTranslatorPB.java | 19 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 136 ++--
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 10 +-
.../hdds/scm/server/StorageContainerManager.java | 44 +-
.../scm/server/upgrade/FinalizationCheckpoint.java | 25 +-
.../scm/server/upgrade/FinalizationManager.java | 16 +
.../server/upgrade/FinalizationManagerImpl.java | 38 +
.../server/upgrade/FinalizationStateManager.java | 15 +-
.../upgrade/FinalizationStateManagerImpl.java | 150 +++-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 88 ++-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 18 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 58 +-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../scm/container/TestContainerManagerImpl.java | 7 +-
.../scm/container/TestContainerReportHandler.java | 25 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../container/balancer/TestContainerBalancer.java | 120 +--
.../container/balancer/TestFindTargetStrategy.java | 3 +-
.../TestSCMContainerPlacementCapacity.java | 10 +
.../TestSCMContainerPlacementRackAware.java | 5 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 96 ++-
.../replication/TestECOverReplicationHandler.java | 5 +
.../replication/TestECUnderReplicationHandler.java | 130 +++-
.../replication/TestLegacyReplicationManager.java | 95 ++-
.../replication/TestOverReplicatedProcessor.java | 124 +++
.../replication/TestReplicationManager.java | 78 +-
.../replication/TestUnderReplicatedProcessor.java | 179 +++++
.../hdds/scm/ha/TestBackgroundSCMService.java | 7 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 16 +-
.../TestPipelineDatanodesIntersection.java | 3 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 8 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 14 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 33 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 2 +-
.../pipeline/TestWritableECContainerProvider.java | 28 +-
.../hdds/scm/update/server/MockCRLStore.java | 4 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 6 +-
.../hdds/scm/upgrade/TestScmFinalization.java | 52 +-
.../org/apache/ozone/test/GenericTestUtils.java | 7 -
.../hdds/scm/cli/ContainerOperationClient.java | 5 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++---
.../apache/hadoop/ozone/client/OzoneClient.java | 13 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +-
.../checksum/AbstractBlockChecksumComputer.java | 5 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 82 +-
.../checksum/ReplicatedFileChecksumHelper.java | 16 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 32 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 30 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 19 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 72 +-
.../TestReplicatedBlockChecksumComputer.java | 56 +-
hadoop-ozone/common/pom.xml | 6 -
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 5 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 102 ++-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 240 ++++++
...vider.java => OMFailoverProxyProviderBase.java} | 304 ++------
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 +
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 7 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 18 +-
.../hadoop/ozone/om/multitenant/AccessPolicy.java | 2 +-
.../hadoop/ozone/om/multitenant/OzoneTenant.java | 10 +-
.../ozone/om/multitenant/RangerAccessPolicy.java | 4 +-
.../apache/hadoop/ozone/om/multitenant/Tenant.java | 12 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 12 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 27 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 20 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 10 +-
.../OMInterServiceProtocolClientSideImpl.java | 12 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 18 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 12 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 347 +++++++++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 508 ++++--------
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 42 +-
.../dist/src/main/compose/ozone/monitoring.conf | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 14 -
.../src/main/compose/ozonesecure/docker-config | 26 +-
.../ozonesecure/mockserverInitialization.json | 98 ---
.../main/compose/upgrade/compose/ha/docker-config | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 47 ++
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../smoketest/security/ozone-secure-tenant.robot | 4 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 13 +-
.../network-tests/src/test/blockade/README.md | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 54 ++
.../TestContainerStateManagerIntegration.java | 2 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 3 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 5 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 4 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 151 +++-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 230 +-----
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 272 +++++++
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 352 +++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 37 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 28 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 6 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 3 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 55 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 262 +++++++
.../commandhandler/TestBlockDeletion.java | 101 +--
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 17 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 6 +-
.../hadoop/ozone/om/OmFailoverProxyUtil.java | 4 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 302 ++++----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 36 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 6 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../ozone/om/multitenant/RangerUserRequest.java | 278 +++++++
...estMultiTenantAccessAuthorizerRangerPlugin.java | 269 -------
.../om/multitenant/TestMultiTenantVolume.java | 29 +
.../om/multitenant/TestRangerBGSyncService.java | 349 ++++-----
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 3 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 6 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 14 +
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 5 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 27 +-
.../TestResetDeletedBlockRetryCountShell.java | 169 ++++
.../src/main/proto/OmClientProtocol.proto | 11 +
hadoop-ozone/ozone-manager/pom.xml | 6 -
.../hadoop/ozone/om/OMMultiTenantManager.java | 112 ++-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 350 +++++----
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 58 +-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 20 +-
.../StringOMHashCodeGeneratorImpl.java | 16 +-
.../ozone/om/hashcodegenerator/package-info.java} | 17 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 27 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 3 +-
.../ozone/om/multitenant/CachedTenantState.java | 41 +
.../InMemoryMultiTenantAccessController.java | 49 +-
.../multitenant/MultiTenantAccessAuthorizer.java | 282 -------
.../MultiTenantAccessAuthorizerDummyPlugin.java | 194 -----
.../MultiTenantAccessAuthorizerRangerPlugin.java | 854 ---------------------
.../multitenant/MultiTenantAccessController.java | 190 +++--
.../om/multitenant/OMRangerBGSyncService.java | 194 +++--
.../RangerClientMultiTenantAccessController.java | 261 +++++--
.../RangerRestMultiTenantAccessController.java | 31 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 +-
.../metrics/OzoneManagerDoubleBufferMetrics.java | 13 +
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 7 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 7 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 8 +
.../request/s3/security/S3SecretRequestHelper.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 2 +
.../protocolPB/OzoneManagerRequestHandler.java | 15 +
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 32 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 126 ++-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 11 +-
.../TestMultiTenantAccessController.java | 97 ++-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +
.../bucket/TestOMBucketSetPropertyRequest.java | 8 +-
.../om/request/key/TestOMKeyCommitRequest.java | 43 ++
.../request/key/TestOMOpenKeysDeleteRequest.java | 71 +-
.../s3/security/TestS3GetSecretRequest.java | 11 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 8 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 17 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 1 +
.../ozone/recon/api/types/DatanodeMetadata.java | 14 +
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 11 +-
.../recon/scm/ReconPipelineReportHandler.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 18 +-
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 72 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 90 +--
.../ozone/recon/api/TestTaskStatusService.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 22 +-
.../recon/fsck/TestContainerHealthStatus.java | 12 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 16 +-
.../TestContainerHealthTaskRecordGenerator.java | 14 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 2 +
.../TestReconInternalSchemaDefinition.java | 21 +-
.../recon/persistence/TestSqlSchemaSetup.java | 6 +-
.../persistence/TestStatsSchemaDefinition.java | 29 +-
.../TestUtilizationSchemaDefinition.java | 40 +-
.../scm/AbstractReconContainerManagerTest.java | 27 +-
.../ozone/recon/scm/TestReconContainerManager.java | 42 +-
...TestReconIncrementalContainerReportHandler.java | 16 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +-
.../recon/scm/TestReconPipelineReportHandler.java | 6 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 21 +-
.../TestStorageContainerServiceProviderImpl.java | 12 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 10 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 9 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 44 +-
.../ozone/recon/tasks/TestTableCountTask.java | 10 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 35 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 131 ++--
.../ozone/s3/signature/AWSSignatureProcessor.java | 30 +-
.../hadoop/ozone/s3/signature/AuthOperation.java} | 36 +-
.../s3/signature/AuthorizationV2HeaderParser.java | 12 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 92 +--
.../s3/signature/AuthorizationV4QueryParser.java | 3 +-
.../hadoop/ozone/s3/signature/Credential.java | 14 +-
...Parser.java => MalformedResourceException.java} | 27 +-
.../hadoop/ozone/s3/signature/SignatureParser.java | 4 +-
.../apache/hadoop/ozone/s3/util/AuditUtils.java | 53 ++
.../hadoop/ozone/client/ClientProtocolStub.java | 573 ++++++++++++++
.../hadoop/ozone/client/ObjectStoreStub.java | 1 +
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/client/OzoneClientStub.java | 7 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 28 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 28 +-
.../signature/TestAuthorizationV2HeaderParser.java | 70 +-
.../signature/TestAuthorizationV4HeaderParser.java | 98 ++-
.../s3/signature/TestStringToSignProducer.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 5 +-
.../ozone/admin/om/UpdateRangerSubcommand.java | 96 +++
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 73 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 48 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 6 +-
.../ozone/debug/container/ContainerCommands.java | 6 +-
.../ozone/freon/ClosedContainerReplicator.java | 33 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 2 +-
.../ozone/freon/OmBucketReadWriteKeyOps.java | 14 +-
.../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 5 +-
pom.xml | 62 +-
457 files changed, 11878 insertions(+), 5997 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org