You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2026/05/15 15:48:24 UTC
(ozone) 01/01: Merge remote-tracking branch 'upstream/master' into HDDS-13323-sts
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch HDDS-13323-sts
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 603a8abc3758bf93ad13b40cb71a11495cb8886f
Merge: 37a224b2176 99328c24239
Author: Fabian Morgan <fm...@gmail.com>
AuthorDate: Tue May 12 20:24:35 2026 -0700
Merge remote-tracking branch 'upstream/master' into HDDS-13323-sts
Conflicts:
hadoop-hdds/common/src/main/resources/ozone-default.xml
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/RequestContext.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketAclHandler.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
.devcontainer/devcontainer.json | 29 +
.github/dependabot.yml | 10 +
.github/pull_request_template.md | 3 +
.github/workflows/build-ratis.yml | 32 +-
.github/workflows/check.yml | 32 +-
.github/workflows/ci-with-ratis.yml | 9 +-
.github/workflows/ci.yml | 67 +-
.github/workflows/close-stale-prs.yaml | 4 +-
.github/workflows/generate-config-doc.yml | 11 +-
.github/workflows/intermittent-test-check.yml | 64 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 14 +-
.github/workflows/post-commit.yml | 7 +-
.github/workflows/pull-request.yml | 6 +-
.github/workflows/repeat-acceptance.yml | 22 +-
.github/workflows/schedule-label-pr.yml | 5 +-
.github/workflows/scheduled-cache-update.yml | 3 +-
.github/workflows/update-ozone-site-config-doc.yml | 45 +-
.github/workflows/{pull-request.yml => zizmor.yml} | 34 +-
.mvn/extensions.xml | 2 +-
CONTRIBUTING.md | 5 +-
README.md | 11 +-
SECURITY.md | 2 +-
dev-support/ci/xml_to_md.py | 31 +-
dev-support/pmd/pmd-ruleset.xml | 7 +
hadoop-hdds/cli-common/pom.xml | 16 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 33 +-
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 20 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 154 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 8 +
.../hdds/scm/storage/StreamBlockInputStream.java | 48 +-
.../io/ECBlockReconstructedStripeInputStream.java | 8 +-
.../scm/storage/TestStreamBlockInputStream.java | 193 +-
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 9 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 -
hadoop-hdds/common/pom.xml | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 22 +
.../org/apache/hadoop/hdds/client/BlockID.java | 4 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 10 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 37 +
.../balancer/ContainerBalancerConfiguration.java | 34 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hadoop/hdds/security/SecurityConfig.java | 36 +
.../apache/hadoop/hdds/tracing/LoopSampler.java} | 28 +-
.../apache/hadoop/hdds/tracing/SkipTracing.java | 18 +-
.../apache/hadoop/hdds/tracing/SpanSampler.java | 90 +
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 31 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 128 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 160 +-
.../apache/hadoop/hdds/utils/SlidingWindow.java | 11 +-
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 21 +-
.../hdds/utils/io/RandomAccessFileChannel.java | 37 +-
.../org/apache/hadoop/ipc_/AlignmentContext.java | 2 +-
.../src/main/java/org/apache/hadoop/ipc_/RPC.java | 18 +-
.../main/java/org/apache/hadoop/ipc_/Server.java | 51 +-
.../org/apache/hadoop/ipc_/WritableRpcEngine.java | 630 --
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 3 +
.../org/apache/hadoop/ozone/common/Checksum.java | 4 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 78 -
.../ozone/common/PureJavaCrc32ByteBuffer.java | 23 +-
.../ozone/common/PureJavaCrc32CByteBuffer.java | 23 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../security_/CustomizedCallbackHandler.java | 121 +
.../hadoop/security_/SaslMechanismFactory.java | 90 +
.../org/apache/hadoop/security_/SaslRpcClient.java | 16 +-
.../org/apache/hadoop/security_/SaslRpcServer.java | 63 +-
.../common/src/main/resources/ozone-default.xml | 230 +-
.../security/TestSecurityConfigTlsSettings.java | 90 +
.../hadoop/hdds/tracing/TestSpanSampling.java | 144 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 34 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 50 +-
.../hdds/utils/io/TestRandomAccessFileChannel.java | 153 +
.../ozone/common/TestChecksumByteBuffer.java | 4 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +-
hadoop-hdds/config/pom.xml | 10 +
.../hadoop/hdds/conf/ConfigFileAppender.java | 8 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 2 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 12 +-
.../ozone/container/common/impl/ContainerData.java | 48 +-
.../container/common/impl/ContainerDataYaml.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 237 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 6 +-
.../states/endpoint/VersionEndpointTask.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 13 +-
.../server/ratis/ContainerStateMachine.java | 22 +-
.../ozone/container/common/volume/HddsVolume.java | 44 +-
.../container/common/volume/MutableVolumeSet.java | 9 +-
.../container/common/volume/StorageVolume.java | 121 +-
.../common/volume/StorageVolumeChecker.java | 114 +-
.../common/volume/ThrottledAsyncChecker.java | 2 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ContainerChoosingPolicyFactory.java | 55 +
.../diskbalancer/DiskBalancerConfiguration.java | 138 +-
.../container/diskbalancer/DiskBalancerInfo.java | 55 +-
.../diskbalancer/DiskBalancerProtocolServer.java | 5 +-
.../diskbalancer/DiskBalancerService.java | 160 +-
.../container/diskbalancer/DiskBalancerYaml.java | 20 +-
...ChoosingPolicy.java => ContainerCandidate.java} | 42 +-
.../policy/ContainerChoosingPolicy.java | 36 +-
.../policy/DefaultContainerChoosingPolicy.java | 208 +-
.../policy/DefaultVolumeChoosingPolicy.java | 156 -
.../ECReconstructionCoordinator.java | 10 -
.../keyvalue/impl/FilePerBlockStrategy.java | 27 +-
.../ozone/container/metadata/AbstractRDBStore.java | 14 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 4 +-
.../container/replication/ReplicationServer.java | 5 +
.../ScmHAFinalizeUpgradeActionDatanode.java | 1 +
.../commands/ReplicateContainerCommand.java | 10 +-
.../container/common/impl/TestHddsDispatcher.java | 43 +
.../statemachine/TestDatanodeConfiguration.java | 32 +
.../server/ratis/TestContainerStateMachine.java | 45 +-
.../common/volume/TestStorageVolumeChecker.java | 185 +
.../volume/TestStorageVolumeHealthChecks.java | 149 +-
.../TestDefaultContainerChoosingPolicy.java | 780 +-
.../TestDefaultVolumeChoosingPolicy.java | 651 --
.../TestDiskBalancerProtocolServer.java | 65 +-
.../diskbalancer/TestDiskBalancerService.java | 81 +-
.../diskbalancer/TestDiskBalancerTask.java | 22 +-
.../diskbalancer/TestDiskBalancerYaml.java | 79 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 51 +
.../TestBackgroundContainerDataScanner.java | 114 +
hadoop-hdds/docs/content/design/diskbalancer.md | 16 +-
.../docs/content/design/s3-conditional-requests.md | 668 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 43 +-
.../docs/content/feature/DiskBalancer.zh.md | 25 +-
.../feature/Snapshot-Configuration-Properties.md | 1 +
hadoop-hdds/docs/content/feature/Snapshot.md | 9 +-
.../content/feature/SnapshotDefragmentation.md | 212 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 23 +-
hadoop-hdds/framework/pom.xml | 13 +-
.../hdds/conf/TracingReconfigurationCallback.java | 58 +
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 12 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 29 +
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHandler.java | 17 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 16 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 37 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 57 +-
.../symmetric/DefaultSecretKeySignerClient.java | 2 +-
.../hdds/security/symmetric/SecretKeyState.java | 9 +-
.../certificate/authority/CertificateStore.java | 8 +-
.../client/DefaultCertificateClient.java | 6 +-
.../certificate/utils/SelfSignedCertificate.java | 2 +-
.../apache/hadoop/hdds/server/OzoneBlacklist.java | 219 +
.../server/OzoneProtocolMessageDispatcher.java | 3 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 33 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 38 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 15 +-
.../apache/hadoop/hdds/utils/HttpServletUtils.java | 2 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 14 +-
.../hdds/utils/db/RDBStoreAbstractIterator.java | 19 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 4 +
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 23 +-
.../org/apache/hadoop/ozone/common/BlockGroup.java | 9 +-
.../apache/hadoop/ozone/common/DeletedBlock.java | 17 +-
.../src/main/resources/webapps/static/ozone.js | 2 +
.../resources/webapps/static/templates/menu.html | 2 +
.../hdds/fs/TestCachingSpaceUsageSource.java | 35 +
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 2 +-
.../hdds/security/ssl/TestGrpcTlsConfig.java | 276 +
.../hdds/server/http/TestHttpServer2SSL.java | 361 +
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 18 +-
.../hdds/utils/db/TestFixedLengthStringCodec.java | 7 +-
.../utils/db/TestRDBStoreIteratorWithDBClose.java | 308 +
.../src/main/proto/ScmAdminProtocol.proto | 28 +
.../src/main/proto/RpcHeader.proto | 2 +-
.../interface-client/src/main/proto/hdds.proto | 16 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../utils/db/managed/ManagedRocksIterator.java | 31 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 5 +
.../scm/block/DeletedBlockLogStateManager.java | 33 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 50 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 39 +-
.../hdds/scm/container/ContainerManagerImpl.java | 40 +-
.../hdds/scm/container/ContainerStateManager.java | 41 +-
.../scm/container/ContainerStateManagerImpl.java | 44 +-
.../scm/container/balancer/ContainerBalancer.java | 20 +-
.../ContainerBalancerSelectionCriteria.java | 107 +-
.../container/balancer/ContainerBalancerTask.java | 2 +
.../hdds/scm/container/balancer/MoveManager.java | 109 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 -
.../container/placement/metrics/SCMMetrics.java | 36 +-
...nQueue.java => MonitoringReplicationQueue.java} | 4 +-
.../QuasiClosedStuckOverReplicationHandler.java | 4 +-
.../replication/QuasiClosedStuckReplicaCount.java | 77 +-
.../QuasiClosedStuckUnderReplicationHandler.java | 13 +-
.../replication/RatisUnderReplicationHandler.java | 9 +
.../container/replication/ReplicationManager.java | 74 +-
.../replication/health/EmptyContainerHandler.java | 74 +-
.../health/QuasiClosedStuckReplicationCheck.java | 20 +-
.../scm/container/states/ContainerStateMap.java | 14 +
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 +
.../apache/hadoop/hdds/scm/ha/ReflectionUtil.java | 67 -
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 10 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 35 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 36 +-
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 10 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 7 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 57 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 48 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 8 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 4 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 328 +
.../DeletedBlockLogStateManagerInvoker.java | 148 +
.../ha/invoker/PipelineStateManagerInvoker.java | 282 +
.../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 82 +
.../scm/ha/invoker/ScmInvokerCodeGenerator.java | 698 ++
.../hadoop/hdds/scm/ha/invoker/package-info.java | 13 +-
.../hadoop/hdds/scm/ha/io/ScmBigIntegerCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmBooleanCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmByteStringCodec.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/io/ScmCodec.java | 3 +-
.../hadoop/hdds/scm/ha/io/ScmCodecFactory.java | 122 +-
.../apache/hadoop/hdds/scm/ha/io/ScmEnumCodec.java | 8 +-
.../hadoop/hdds/scm/ha/io/ScmIntegerCodec.java | 10 +-
.../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java | 84 +-
.../apache/hadoop/hdds/scm/ha/io/ScmLongCodec.java | 10 +-
.../hdds/scm/ha/io/ScmManagedSecretKeyCodec.java | 10 +-
.../scm/ha/io/ScmNonShadedByteStringCodec.java | 2 +-
.../ha/io/ScmNonShadedGeneratedMessageCodec.java | 10 +-
.../hadoop/hdds/scm/ha/io/ScmStringCodec.java | 2 +-
.../hdds/scm/ha/io/ScmX509CertificateCodec.java | 8 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 10 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 39 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 14 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 13 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hdds/scm/node/PendingContainerTracker.java | 225 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 30 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 24 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 -
.../hdds/scm/pipeline/PipelineStateManager.java | 14 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 5 +-
...inerLocationProtocolServerSideTranslatorPB.java | 53 +-
.../hdds/scm/security/RootCARotationHandler.java | 9 +-
.../scm/security/RootCARotationHandlerImpl.java | 4 +-
.../hdds/scm/security/RootCARotationManager.java | 17 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 109 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 20 +-
.../hdds/scm/server/StorageContainerManager.java | 27 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/server/StorageContainerManagerStarter.java | 2 -
.../server/upgrade/FinalizationStateManager.java | 9 +-
.../upgrade/FinalizationStateManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 5 +-
.../main/resources/webapps/scm/ratis-events.html} | 36 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../src/main/resources/webapps/scm/scm.js | 26 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 1 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 37 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../scm/container/TestContainerManagerImpl.java | 19 +-
.../scm/container/TestContainerReportHandler.java | 101 +-
.../scm/container/TestContainerStateManager.java | 44 +-
.../TestContainerBalancerSelectionCriteria.java | 188 +-
.../scm/container/balancer/TestMoveManager.java | 175 +
.../algorithms/TestContainerPlacementFactory.java | 2 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 24 +
...TestQuasiClosedStuckOverReplicationHandler.java | 96 +-
.../TestQuasiClosedStuckReplicaCount.java | 302 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 61 +-
.../health/TestEmptyContainerHandler.java | 36 +
.../TestQuasiClosedStuckReplicationCheck.java | 40 +-
.../container/states/TestContainerStateMap.java | 76 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 19 -
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 4 +-
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 +
.../hdds/scm/ha/io/TestScmBigIntegerCodec.java | 2 +-
.../hadoop/hdds/scm/ha/io/TestScmListCodec.java} | 30 +-
.../scm/ha/io/TestScmX509CertificateCodec.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 185 +
.../hdds/scm/node/TestPendingContainerTracker.java | 381 +
.../hdds/scm/node/states/TestNodeStateMap.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 11 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 56 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +-
.../scm/security/TestRootCARotationManager.java | 2 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 3 +-
.../scm/server/TestSCMClientProtocolServer.java | 31 +-
.../replicationManagerTests/quasi_closed.json | 57 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 6 +-
.../apache/ozone/test/TimedOutTestsListener.java | 4 +-
hadoop-ozone/cli-admin/pom.xml | 13 +
.../scm/cli/ContainerBalancerStatusSubcommand.java | 21 +-
.../hdds/scm/cli/ContainerOperationClient.java | 21 +
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 5 +-
.../scm/cli/container/ContainerIDParameters.java | 14 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 12 +-
.../hdds/scm/cli/container/ListSubcommand.java | 9 +-
.../scm/cli/container/ReconcileSubcommand.java | 26 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 65 +-
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 54 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 6 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 4 +-
.../cli/datanode/DiskBalancerReportSubcommand.java | 145 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 11 +
.../cli/datanode/DiskBalancerStatusSubcommand.java | 23 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 15 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 62 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 4 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 14 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 11 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 4 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 13 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 8 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 2 +-
.../ozone/admin/om/ListOpenFilesSubCommand.java | 61 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 18 +-
.../ozone/admin/scm/RotateKeySubCommand.java | 2 +-
.../TestContainerReportSuppressOptions.java | 310 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 5 +-
.../scm/cli/container/TestReconcileSubcommand.java | 50 +-
.../datanode/TestContainerBalancerSubCommand.java | 2 +-
.../cli/datanode/TestDiskBalancerSubCommands.java | 88 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 62 +-
hadoop-ozone/cli-debug/pom.xml | 20 +-
.../ozone/debug/audit/parser/AuditParser.java | 4 +-
.../debug/audit/parser/common/DatabaseHelper.java | 4 +-
.../audit/parser/handler/LoadCommandHandler.java | 6 +-
.../audit/parser/handler/QueryCommandHandler.java | 6 +-
.../parser/handler/TemplateCommandHandler.java | 16 +-
.../ozone/debug/kerberos/AuthorizationProbe.java | 106 +
.../hadoop/ozone/debug/kerberos/ConfigProbe.java | 116 +
.../ozone/debug/kerberos/DiagnoseSubcommand.java | 113 +
.../ozone/debug/kerberos/DiagnosticProbe.java} | 24 +-
.../ozone/debug/kerberos/EnvironmentProbe.java | 27 +-
.../hadoop/ozone/debug/kerberos/HostProbe.java | 72 +
.../hadoop/ozone/debug/kerberos/HttpAuthProbe.java | 52 +
.../ozone/debug/kerberos/JvmKerberosProbe.java | 98 +
.../ozone/debug/kerberos/KerberosConfigProbe.java | 61 +
.../ozone/debug/kerberos/KerberosSubcommand.java | 28 +-
.../ozone/debug/kerberos/KerberosTicketProbe.java | 95 +
.../hadoop/ozone/debug/kerberos/KeytabProbe.java | 93 +
.../hadoop/ozone/debug/kerberos/KinitProbe.java | 61 +
.../debug/kerberos/PrincipalMappingProbe.java | 113 +
.../hadoop/ozone/debug/kerberos/ProbeResult.java} | 14 +-
.../ozone/debug/kerberos/SecurityConfigProbe.java | 72 +
.../kerberos/TranslatePrincipalSubcommand.java | 78 +
.../hadoop/ozone/debug/kerberos/package-info.java} | 15 +-
.../debug/kerberos/TestOzoneDebugKerberos.java | 159 +
.../hadoop/ozone/debug/kerberos/package-info.java} | 12 +-
hadoop-ozone/cli-repair/pom.xml | 16 +-
.../schemaupgrade/ContainerUpgradeResult.java | 36 +-
.../schemaupgrade/VolumeUpgradeResult.java | 28 +-
hadoop-ozone/cli-shell/pom.xml | 16 +-
.../hadoop/ozone/shell/ListPaginationOptions.java | 2 +-
.../hadoop/ozone/shell/OzoneInteractiveShell.java | 84 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 26 +-
.../apache/hadoop/ozone/shell/acl/AclOption.java | 4 +-
.../hadoop/ozone/shell/bucket/BucketUri.java | 2 +-
.../ozone/shell/bucket/DeleteBucketHandler.java | 4 +-
.../hadoop/ozone/shell/common/VolumeBucketUri.java | 2 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 2 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 10 +-
.../hadoop/ozone/shell/prefix/PrefixUri.java | 2 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 43 +-
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 2 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 2 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +-
.../hadoop/ozone/shell/volume/VolumeUri.java | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 49 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 111 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 16 +
.../ozone/client/io/KeyDataStreamOutput.java | 6 +-
.../ozone/client/protocol/ClientProtocol.java | 138 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 249 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 9 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 36 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 4 +-
.../hadoop/ozone/client/checksum/CrcComposer.java | 79 +-
.../hadoop/ozone/client/checksum/CrcUtil.java | 155 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +-
.../hadoop/ozone/om/exceptions/OMException.java | 9 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 55 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 32 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 166 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 349 +
.../ozone/om/helpers/OmMultipartPartKey.java | 233 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 4 +-
.../ozone/om/helpers/OzoneFileStatusLight.java | 4 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 21 +
.../apache/hadoop/ozone/om/helpers/S3STSUtils.java | 16 +-
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 15 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 50 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 9 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 61 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 103 +-
.../ozone/security/acl/AssumeRoleRequest.java | 28 +-
.../hadoop/ozone/security/acl/RequestContext.java | 20 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 59 +-
.../ozone/snapshot/SubmitSnapshotDiffResponse.java | 65 +
.../org/apache/hadoop/ozone/util/RadixTree.java | 4 +-
.../ozone/client/checksum/TestCrcComposer.java | 222 +
.../hadoop/ozone/client/checksum/TestCrcUtil.java | 236 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 41 +
.../ozone/om/helpers/TestOmMultipartPartInfo.java | 143 +
.../ozone/om/helpers/TestOmMultipartPartKey.java | 204 +
.../om/helpers/TestOmSnapshotDiffJobCodec.java | 2 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 24 +
.../ozone/security/acl/TestAssumeRoleRequest.java | 73 +-
.../acl/iam/TestIamSessionPolicyResolver.java | 26 +-
.../hadoop/ozone/security/acl/package-info.java} | 12 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 78 +
.../snapshot/TestSubmitSnapshotDiffResponse.java | 97 +
hadoop-ozone/dist/README.md | 6 +-
hadoop-ozone/dist/pom.xml | 15 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +
.../dist/src/main/compose/common/hadoop-test.sh | 5 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 2 +-
.../src/main/compose/ozone-om-ha/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 4 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +-
.../compose/ozonesecure-ha/test-debug-tools.sh | 2 +
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 26 +
.../main/compose/upgrade/compose/ha/docker-config | 1 +
.../compose/upgrade/compose/non-ha/docker-config | 1 +
.../dist/src/main/compose/xcompat/new-cluster.yaml | 3 +
.../dist/src/main/compose/xcompat/old-cluster.yaml | 3 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 7 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 8 +-
.../dist/src/main/smoketest/cli/classpath.robot | 47 +-
.../src/main/smoketest/compatibility/setup.robot | 9 +-
.../src/main/smoketest/compatibility/write.robot | 65 +
.../smoketest/debug/ozone-debug-kerberos.robot | 59 +
.../omha/data-validation-after-om-bootstrap.robot | 15 +-
.../src/main/smoketest/s3/conditionalput.robot | 89 +
.../main/smoketest/scmha/container-create.robot | 2 +-
.../main/smoketest/snapshot/snapshot-defrag.robot | 172 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 12 +
.../dist/src/shell/ozone/ozone-functions.sh | 16 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 5 -
.../network-tests/src/test/compose/docker-config | 4 +-
hadoop-ozone/freon/pom.xml | 20 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 18 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 143 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 2 +-
hadoop-ozone/httpfsgateway/pom.xml | 32 -
.../src/main/resources/httpfs-default.xml | 8 -
.../iceberg/dev-support/findbugsExcludeFile.xml | 16 +
hadoop-ozone/iceberg/pom.xml | 105 +
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 463 +
.../ozone/iceberg/RewriteTablePathOzoneUtils.java | 126 +
.../apache/hadoop/ozone/iceberg/package-info.java} | 12 +-
.../iceberg/TestRewriteTablePathOzoneAction.java | 361 +
hadoop-ozone/integration-test-recon/pom.xml | 15 +
...> AbstractTestStorageDistributionEndpoint.java} | 282 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 1 +
.../ozone/recon/TestReconContainerEndpoint.java | 80 +-
.../ozone/recon/TestReconOmMetaManagerUtils.java | 33 +
.../apache/hadoop/ozone/recon/TestReconTasks.java | 802 +-
.../ozone/recon/TestReconTasksMultiNode.java | 178 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +
.../recon/TestStorageDistributionEndpointEC.java | 93 +
.../TestStorageDistributionEndpointRatis.java | 100 +
hadoop-ozone/integration-test-s3/pom.xml | 17 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 464 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 698 +-
.../src/test/resources/log4j.properties | 0
hadoop-ozone/integration-test/pom.xml | 25 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 20 +-
.../fs/contract/AbstractContractConcatTest.java | 8 +-
.../AbstractContractContentSummaryTest.java | 4 +-
.../AbstractContractCopyFromLocalTest.java | 10 +-
.../AbstractContractGetFileStatusTest.java | 10 +-
.../AbstractContractLeaseRecoveryTest.java | 14 +-
.../AbstractContractMultipartUploaderTest.java | 55 +-
.../fs/contract/AbstractContractOpenTest.java | 74 +-
.../AbstractContractRootDirectoryTest.java | 18 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 306 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 299 +-
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 323 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 144 +
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 51 +
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 1 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 8 +-
.../hdds/scm/TestStorageContainerManager.java | 11 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 27 +
.../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +-
.../TestReplicationManagerIntegration.java | 154 +
.../TestScmDataDistributionFinalization.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 8 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 333 +-
.../rpc/TestContainerStateMachineFailures.java | 65 +-
.../replication/TestContainerReplication.java | 53 +
.../TestContainerCommandReconciliation.java | 3 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 6 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 1 +
...stOzoneManagerHAFollowerReadWithAllRunning.java | 5 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 82 +
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 2 +
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../ozone/reconfig/TestOmReconfiguration.java | 74 +
.../ozone/reconfig/TestScmReconfiguration.java | 2 +
.../scm/node/TestContainerChoosingPolicy.java | 300 -
.../node/TestDiskBalancerPolicyPerformance.java | 381 +
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 -
.../ozone/shell/TestDeletedBlocksTxnShell.java | 3 +-
.../tools/contract/AbstractContractDistCpTest.java | 14 +-
.../src/test/resources/log4j.properties | 6 -
.../src/main/proto/OmClientProtocol.proto | 51 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +
hadoop-ozone/ozone-manager/pom.xml | 15 -
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 5 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 24 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 21 +
.../om/OMDBCheckpointServletInodeBasedXfer.java | 14 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 61 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 12 +-
.../hadoop/ozone/om/OmSnapshotLocalData.java | 2 +-
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 15 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 61 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 266 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 -
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 57 +-
.../hadoop/ozone/om/SnapshotListJSONServlet.java | 96 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 28 +-
.../hadoop/ozone/om/ha/OMPeriodicMetrics.java | 131 +
.../ozone/om/ratis/OzoneManagerStateMachine.java | 49 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 34 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 27 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 96 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 7 +
.../request/snapshot/OMSnapshotRenameRequest.java | 10 +
.../ozone/om/service/DirectoryDeletingService.java | 30 +-
.../ozone/om/service/KeyDeletingService.java | 8 +-
.../om/snapshot/OMSnapshotDirectoryMetrics.java | 328 +
.../om/snapshot/OmSnapshotLocalDataManager.java | 25 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 37 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 214 +-
.../om/snapshot/SnapshotDiffManagerMXBean.java} | 21 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 62 +-
.../hadoop/ozone/security/STSTokenIdentifier.java | 12 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 47 +
.../main/resources/webapps/ozoneManager/index.html | 4 +-
.../webapps/ozoneManager/om-snapshots.html | 176 +
.../resources/webapps/ozoneManager/ozoneManager.js | 155 +
.../webapps/ozoneManager/ratis-events.html | 36 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 80 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 2 +
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 50 +-
.../TestOzoneManagerListMultipartUploadsAcls.java | 196 +
.../ozone/om/TestOzoneManagerListPartsAcls.java | 196 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 55 +-
.../ozone/om/TestSnapshotListJSONServlet.java | 57 +
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 25 +
.../TestOmRatisSnapshotProvider.java | 8 +-
.../request/TestOMClientRequestWithUserInfo.java | 2 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +-
.../request/bucket/TestOMBucketCreateRequest.java | 35 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 77 +-
.../om/request/key/TestOMKeyCreateRequest.java | 235 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 +-
.../s3/security/TestS3GetSecretRequest.java | 2 +-
.../s3/tenant/TestOMTenantCreateRequest.java | 2 +-
.../s3/tenant/TestOMTenantDeleteRequest.java | 2 +-
.../snapshot/TestOMSnapshotRenameRequest.java | 20 +
.../validation/TestOMValidatorProcessor.java | 16 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +-
.../om/service/TestDirectoryDeletingService.java | 94 +-
.../ozone/om/service/TestKeyDeletingService.java | 37 +-
.../om/service/TestSnapshotDiffCleanupService.java | 2 +-
.../snapshot/TestOMSnapshotDirectoryMetrics.java | 125 +
.../snapshot/TestOmSnapshotLocalDataManager.java | 10 +
.../ozone/om/snapshot/TestSnapshotCache.java | 143 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 317 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 156 +
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 87 +
.../ozone/security/acl/TestOzoneBlacklist.java | 235 +
.../ozone/security/acl/TestRequestContext.java | 6 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 61 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 67 +-
hadoop-ozone/pom.xml | 9 +
.../ozone/recon/codegen/TableNamingStrategy.java | 12 +-
.../recon/schema/ContainerSchemaDefinition.java | 20 +-
hadoop-ozone/recon/pom.xml | 14 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 71 +
.../ozone/recon/api/ClusterStateEndpoint.java | 7 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 268 +-
.../hadoop/ozone/recon/api/ExportJobManager.java | 404 +
.../recon/api/StorageDistributionEndpoint.java | 69 +-
.../ozone/recon/api/handlers/BucketHandler.java | 14 +-
.../hadoop/ozone/recon/api/types/ExportJob.java | 222 +
.../ozone/recon/api/types/GlobalStorageReport.java | 20 +-
.../ozone/recon/api/types/UsedSpaceBreakDown.java | 12 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 793 +-
.../fsck/NoOpsContainerReplicaPendingOps.java | 112 +
.../ozone/recon/fsck/ReconReplicationManager.java | 661 ++
.../recon/fsck/ReconReplicationManagerReport.java | 66 +
.../recon/metrics/ContainerHealthMetrics.java | 86 -
.../recon/metrics/ContainerHealthTaskMetrics.java | 75 +
.../recon/metrics/OzoneManagerSyncMetrics.java | 2 +-
.../persistence/ContainerHealthSchemaManager.java | 596 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 128 +-
.../recon/scm/ReconStorageContainerSyncHelper.java | 113 +
.../recon/spi/StorageContainerServiceProvider.java | 8 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 9 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 16 +-
.../impl/StorageContainerServiceProviderImpl.java | 9 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 3 +-
.../ozone/recon/upgrade/ReconLayoutFeature.java | 4 +-
...ntainersStateContainerIdIndexUpgradeAction.java | 86 +
.../webapps/recon/ozone-recon-web/.eslintrc.json | 180 +-
.../webapps/recon/ozone-recon-web/api/db.json | 2736 ++++-
.../recon/ozone-recon-web/api/pagination.js | 86 +
.../webapps/recon/ozone-recon-web/api/routes.json | 5 +
.../webapps/recon/ozone-recon-web/package.json | 10 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 10840 +++++++++++--------
.../mocks/capacityMocks/capacityResponseMocks.ts | 4 +-
.../src/v2/components/plots/insightsFilePlot.tsx | 18 +-
.../src/v2/components/select/multiSelect.tsx | 134 +-
.../v2/components/select/multiSelectMenuList.tsx | 160 +
.../src/v2/components/select/singleSelect.tsx | 1 +
.../src/v2/components/tables/bucketsTable.tsx | 2 +
.../src/v2/components/tables/containersTable.tsx | 71 +-
.../src/v2/components/tables/datanodesTable.tsx | 1 +
.../src/v2/components/tables/pipelinesTable.tsx | 2 +-
.../src/v2/constants/capacity.constants.tsx | 4 +-
.../src/v2/constants/select.constants.tsx | 7 +-
.../src/v2/pages/buckets/buckets.tsx | 9 +-
.../src/v2/pages/capacity/capacity.tsx | 4 +-
.../src/v2/pages/containers/containers.tsx | 816 +-
.../src/v2/pages/insights/insights.tsx | 1 +
.../ozone-recon-web/src/v2/types/capacity.types.ts | 4 +-
.../src/v2/types/container.types.ts | 56 +-
.../src/views/insights/insights.tsx | 12 +-
.../ozone/recon/api/TestClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/TestContainerEndpoint.java | 42 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 2 -
.../ozone/recon/api/TestExportJobManager.java | 432 +
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 2 -
.../ozone/recon/api/TestOpenContainerCount.java | 2 -
.../recon/api/TestOpenKeysSearchEndpoint.java | 2 -
.../recon/api/TestStorageDistributionEndpoint.java | 16 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 2 -
.../ozone/recon/api/types/TestExportJob.java | 110 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 799 +-
.../TestContainerHealthTaskRecordGenerator.java | 710 --
.../recon/fsck/TestReconReplicationManager.java | 515 +
.../ozone/recon/heatmap/TestHeatMapInfo.java | 2 -
.../recon/metrics/TestOzoneManagerSyncMetrics.java | 50 +
.../TestUnhealthyContainersDerbyPerformance.java | 765 ++
.../scm/TestReconStorageContainerSyncHelper.java | 185 +
...tOmTableInsightTaskStaleCounterAfterReinit.java | 243 +
...ntainersStateContainerIdIndexUpgradeAction.java | 117 +
hadoop-ozone/s3gateway/pom.xml | 7 +
.../hadoop/ozone/s3/AuthorizationFilter.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 5 +-
.../ozone/s3/commontypes/EncodingTypeObject.java | 5 +-
.../endpoint/AuditingBucketOperationHandler.java | 70 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 130 +-
.../ozone/s3/endpoint/BucketCrudHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 214 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 5 +
.../s3/endpoint/BucketOperationHandlerChain.java | 95 +
...CompleteMultipartUploadRequestUnmarshaller.java | 9 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 163 +-
.../s3/endpoint/ListMultipartUploadsHandler.java | 92 +
.../ozone/s3/endpoint/MessageUnmarshaller.java | 8 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 2 +
.../ozone/s3/endpoint/MultipartKeyHandler.java | 2 -
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 272 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 40 +-
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 9 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 86 +-
.../ozone/s3/endpoint/S3ConditionalRequest.java | 291 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 39 +-
.../hadoop/ozone/s3/exception/OSTSException.java | 21 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 311 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 9 +-
.../ozone/s3/signature/StringToSignProducer.java | 35 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 11 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 34 +-
.../apache/hadoop/ozone/s3sts/S3STSEndpoint.java | 70 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 73 +
.../hadoop/ozone/client/ObjectStoreStub.java | 54 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 95 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 16 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 5 +
.../ozone/s3/endpoint/EndpointTestUtils.java | 9 +-
.../ozone/s3/endpoint/TestBucketAclHandler.java | 82 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 3 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 88 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 85 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 236 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 8 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 32 +-
.../ozone/s3/exception/TestOS3Exceptions.java | 10 +-
.../ozone/s3/exception/TestOSTSExceptions.java | 8 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 9 +-
.../s3/signature/TestAWSSignatureProcessor.java | 23 +-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 41 +
hadoop-ozone/tools/pom.xml | 22 +-
.../ozone/local/LocalOzoneClusterConfig.java | 327 +
.../hadoop/ozone/local/LocalOzoneRuntime.java | 81 +
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 278 +
.../apache/hadoop/ozone/local/package-info.java} | 12 +-
.../ozone/local/TestLocalOzoneClusterConfig.java | 135 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 365 +
hadoop-ozone/vapor/pom.xml | 16 +-
.../containergenerator/GeneratorDatanode.java | 4 +-
pom.xml | 315 +-
.../FileSystem/failure_injector_fs.cc | 2 +-
779 files changed, 46940 insertions(+), 15928 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index 2066a99baa4,f554b440602..e844447cd4b
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -5051,25 -5068,16 +5116,36 @@@
<value>5m</value>
<description>Interval for cleaning up orphan snapshot local data versions corresponding to snapshots</description>
</property>
-
+ <property>
+ <name>ozone.om.sts.token.cleanup.service.interval</name>
+ <value>3h</value>
+ <tag>OZONE, OM, PERFORMANCE, SECURITY</tag>
+ <description>
+ A background job that periodically checks revoked STS token entries and
+ deletes ones that have existed for 12 hours. This entry controls the interval of this
+ cleanup check. Unit could be defined with postfix (ns,ms,s,m,h,d).
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.sts.token.cleanup.service.timeout</name>
+ <value>15m</value>
+ <tag>OZONE, OM, PERFORMANCE, SECURITY</tag>
+ <description>
+ A timeout value for the revoked STS token cleanup service. If this is set
+ greater than 0, the service will stop waiting for the deletion
+ completion after this time. Unit could be defined with postfix (ns,ms,s,m,h,d).
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, SCM</tag>
+ <description>The maximum number of events that can be pending in SCM Ratis.</description>
+ </property>
+ <property>
+ <name>ozone.om.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, OM</tag>
+ <description>The maximum number of events that can be pending in OM Ratis.</description>
+ </property>
</configuration>
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 445355987d7,ea17ad9ca85..0a2d506fd69
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -687,16 -699,10 +699,20 @@@ public final class OMConfigKeys
"ozone.om.snapshot.local.data.manager.service.interval";
public static final String OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT = "5m";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_INTERVAL
+ = "ozone.om.sts.token.cleanup.service.interval";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_INTERVAL_DEFAULT
+ = "3h";
+
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_TIMEOUT
+ = "ozone.om.sts.token.cleanup.service.timeout";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_TIMEOUT_DEFAULT
+ = "15m";
+
+ public static final String OZONE_OM_RATIS_EVENTS_MAX_LIMIT =
+ "ozone.om.ratis.events.max.limit";
+ public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
+
/**
* Never constructed.
*/
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index fcd1e7807f1,240e99e7d67..39e1042b474
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -276,7 -276,10 +276,12 @@@ public class OMException extends IOExce
TOO_MANY_SNAPSHOTS,
+ ETAG_MISMATCH,
+
+ ETAG_NOT_AVAILABLE,
+
+ ATOMIC_WRITE_CONFLICT,
+ REVOKED_TOKEN,
+ MALFORMED_POLICY_DOCUMENT,
}
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index b5cdcc52540,5d2de09c48e..3d29637d973
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@@ -61,9 -61,7 +61,10 @@@ public final class OmKeyArgs extends Wi
// This allows a key to be created an committed atomically if the original has not
// been modified.
private Long expectedDataGeneration = null;
+ // Original S3/list prefix when keyName is empty (root listing). Used for STS
+ // auth to check LIST on this prefix instead of "*".
+ private final String listPrefix;
+ private final String expectedETag;
private OmKeyArgs(Builder b) {
super(b);
@@@ -85,7 -83,7 +86,8 @@@
this.ownerName = b.ownerName;
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
+ this.listPrefix = b.listPrefix;
+ this.expectedETag = b.expectedETag;
}
public boolean getIsMultipartKey() {
@@@ -168,14 -166,10 +170,18 @@@
return expectedDataGeneration;
}
+ /**
+ * Original S3/list prefix when keyName is empty (root listing).
+ * Used for STS auth to check LIST on this prefix instead of "*".
+ */
+ public String getListPrefix() {
+ return listPrefix;
+ }
+
+ public String getExpectedETag() {
+ return expectedETag;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@@ -246,7 -243,7 +255,8 @@@
private boolean forceUpdateContainerCacheFromSCM;
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
+ private String listPrefix = null;
+ private String expectedETag;
public Builder() {
this(AclListBuilder.empty());
@@@ -276,9 -289,9 +302,10 @@@
this.forceUpdateContainerCacheFromSCM =
obj.forceUpdateContainerCacheFromSCM;
this.expectedDataGeneration = obj.expectedDataGeneration;
+ this.expectedETag = obj.expectedETag;
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
+ this.listPrefix = obj.listPrefix;
}
public Builder setVolumeName(String volume) {
@@@ -412,11 -425,11 +439,16 @@@
return this;
}
+ public Builder setListPrefix(String prefix) {
+ this.listPrefix = prefix;
+ return this;
+ }
+
+ public Builder setExpectedETag(String eTag) {
+ this.expectedETag = eTag;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(this);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
index c70c01a8723,00000000000..763c8fe9bfa
mode 100644,000000..100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
@@@ -1,170 -1,0 +1,170 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+
+import com.google.common.base.Strings;
+import java.util.Map;
+import net.jcip.annotations.Immutable;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+
+/**
+ * Utility class containing constants and validation methods shared by STS endpoint and OzoneManager processing.
+ */
+@Immutable
+public final class S3STSUtils {
+ // STS API constants
+ public static final int DEFAULT_DURATION_SECONDS = 3600; // 1 hour
+ public static final int MAX_DURATION_SECONDS = 43200; // 12 hours
+ public static final int MIN_DURATION_SECONDS = 900; // 15 minutes
+
+ public static final int ASSUME_ROLE_SESSION_NAME_MIN_LENGTH = 2;
+ public static final int ASSUME_ROLE_SESSION_NAME_MAX_LENGTH = 64;
+
+ // AWS limit for session policy is 2048 characters
+ public static final int MAX_SESSION_POLICY_LENGTH = 2048;
+
+ private S3STSUtils() {
+ }
+
+ /**
+ * Adds standard AssumeRole audit params.
+ */
+ public static void addAssumeRoleAuditParams(Map<String, String> auditParams, String roleArn, String roleSessionName,
+ String awsIamSessionPolicy, int duration, String requestId) {
+
+ auditParams.put("action", "AssumeRole");
+ auditParams.put("roleArn", roleArn);
+ auditParams.put("roleSessionName", roleSessionName);
+ auditParams.put("duration", String.valueOf(duration));
+ auditParams.put("isPolicyIncluded", Strings.isNullOrEmpty(awsIamSessionPolicy) ? "N" : "Y");
+ auditParams.put("requestId", requestId);
+ }
+
+ /**
+ * Validates the duration in seconds.
+ * @param durationSeconds duration in seconds
+ * @return validated duration
+ * @throws OMException if duration is invalid
+ */
+ public static int validateDuration(Integer durationSeconds) throws OMException {
+ if (durationSeconds == null) {
+ return DEFAULT_DURATION_SECONDS;
+ }
+
+ if (durationSeconds < MIN_DURATION_SECONDS || durationSeconds > MAX_DURATION_SECONDS) {
+ throw new OMException(
+ "Invalid Value: DurationSeconds must be between " + MIN_DURATION_SECONDS + " and " + MAX_DURATION_SECONDS +
+ " seconds", INVALID_REQUEST);
+ }
+
+ return durationSeconds;
+ }
+
+ /**
+ * Validates the role session name.
+ * @param roleSessionName role session name
+ * @throws OMException if role session name is invalid
+ */
+ public static void validateRoleSessionName(String roleSessionName) throws OMException {
+ if (Strings.isNullOrEmpty(roleSessionName)) {
+ throw new OMException(
+ "Value null at 'roleSessionName' failed to satisfy constraint: Member must not be null", INVALID_REQUEST);
+ }
+
+ final int roleSessionNameLength = roleSessionName.length();
+ if (roleSessionNameLength < ASSUME_ROLE_SESSION_NAME_MIN_LENGTH ||
+ roleSessionNameLength > ASSUME_ROLE_SESSION_NAME_MAX_LENGTH) {
+ throw new OMException("Invalid RoleSessionName length " + roleSessionNameLength + ": it must be " +
+ ASSUME_ROLE_SESSION_NAME_MIN_LENGTH + "-" + ASSUME_ROLE_SESSION_NAME_MAX_LENGTH + " characters long and " +
+ "contain only alphanumeric characters and +, =, ,, ., @, -", INVALID_REQUEST);
+ }
+
+ // AWS allows: alphanumeric, +, =, ,, ., @, -
+ // Pattern: [\w+=,.@-]*
+ // Don't use regex for performance reasons
+ for (int i = 0; i < roleSessionNameLength; i++) {
+ final char c = roleSessionName.charAt(i);
+ if (!isRoleSessionNameChar(c)) {
+ throw new OMException("Invalid character '" + c + "' in RoleSessionName: it must be " +
+ ASSUME_ROLE_SESSION_NAME_MIN_LENGTH + "-" + ASSUME_ROLE_SESSION_NAME_MAX_LENGTH + " characters long and " +
+ "contain only alphanumeric characters and +, =, ,, ., @, -", INVALID_REQUEST);
+ }
+ }
+ }
+
+ private static boolean isRoleSessionNameChar(char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
+ c == '_' || c == '+' || c == '=' || c == ',' || c == '.' || c == '@' || c == '-';
+ }
+
+ /**
+ * Validates the session policy length.
+ * @param awsIamSessionPolicy session policy
+ * @throws OMException if policy length is invalid
+ */
+ public static void validateSessionPolicy(String awsIamSessionPolicy) throws OMException {
+ if (awsIamSessionPolicy != null && awsIamSessionPolicy.length() > MAX_SESSION_POLICY_LENGTH) {
+ throw new OMException(
+ "Value '" + awsIamSessionPolicy + "' at 'policy' failed to satisfy constraint: Member " +
+ "must have length less than or equal to " + MAX_SESSION_POLICY_LENGTH, INVALID_REQUEST);
+ }
+ }
+
+ /**
+ * Generates the assumed role user ARN.
+ * @param validRoleArn valid role ARN
+ * @param roleSessionName role session name
+ * @return assumed role user ARN
+ */
+ public static String toAssumedRoleUserArn(String validRoleArn, String roleSessionName) {
+ // We already know the roleArn is valid, so perform the conversion for assumed role user arn format
+ // RoleArn format: arn:aws:iam::<account-id>:role/<role-name>
+ // Assumed role user arn format: arn:aws:sts::<account-id>:assumed-role/<role-name>/<role-session-name>
+ final String[] parts = splitRoleArnWithoutRegex(validRoleArn);
+
+ final String partition = parts[1];
+ final String accountId = parts[4];
+ final String resource = parts[5];
+ final String roleName = resource.substring("role/".length());
+
+ //noinspection StringBufferReplaceableByString
- final StringBuilder stringBuilder = new StringBuilder("arn:");
- stringBuilder.append(partition);
- stringBuilder.append(":sts::");
- stringBuilder.append(accountId);
- stringBuilder.append(":assumed-role/");
- stringBuilder.append(roleName);
- stringBuilder.append('/');
- stringBuilder.append(roleSessionName);
++ final StringBuilder stringBuilder = new StringBuilder("arn:")
++ .append(partition)
++ .append(":sts::")
++ .append(accountId)
++ .append(":assumed-role/")
++ .append(roleName)
++ .append('/')
++ .append(roleSessionName);
+ return stringBuilder.toString();
+ }
+
+ private static String[] splitRoleArnWithoutRegex(String roleArn) {
+ final String[] parts = new String[6];
+ int start = 0;
+ for (int i = 0; i < 5; i++) {
+ final int end = roleArn.indexOf(':', start);
+ parts[i] = roleArn.substring(start, end);
+ start = end + 1;
+ }
+ parts[5] = roleArn.substring(start);
+ return parts;
+ }
+}
diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
index 1d976514632,00000000000..f8d37a113d4
mode 100644,000000..100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
@@@ -1,2256 -1,0 +1,2256 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security.acl.iam;
+
+import static java.util.Collections.emptySet;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
+import static org.apache.hadoop.ozone.security.acl.AssumeRoleRequest.OzoneGrant;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.CREATE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.DELETE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.LIST;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ_ACL;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE_ACL;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.NATIVE;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.RANGER;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3ResourceType;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.buildCaseInsensitiveS3ActionMap;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.createPathsAndPermissions;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.groupObjectsByAcls;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.mapPolicyActionsToS3Actions;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.resolve;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.validateAndCategorizeResources;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.security.acl.IOzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for {@link IamSessionPolicyResolver}.
+ * */
+public class TestIamSessionPolicyResolver {
+
+ private static final String VOLUME = "s3v";
+
+ @Test
+ public void testUnsupportedConditionOperatorThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringNotEqualsIgnoreCase\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Condition operator - StringNotEqualsIgnoreCase",
+ NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testUnsupportedConditionAttributeThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"aws:SourceArn\": \"arn:aws:s3:::d\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Condition key name - aws:SourceArn", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testUnsupportedEffectThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Deny\",\n" + // unsupported effect
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::proj-*\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Effect - Deny", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testInvalidJsonWithoutStatementThrows() {
+ final String json = "{\n" +
+ " \"RandomAttribute\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid policy JSON - missing Statement", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidEffectThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": [\"Allow\"],\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Effect in JSON policy (must be a String) - [\"Allow\"]",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testMissingEffectInStatementThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Effect is missing from JSON policy", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidNumberOfConditionsThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": [\n" +
+ " {\n" +
+ " \"StringEquals\": {\n" +
+ " \"aws:SourceArn\": \"arn:aws:s3:::d\"\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"StringEquals\": {\n" +
+ " \"aws:SourceArn\": \"arn:aws:s3:::e\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Only one Condition is supported", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testInvalidConditionThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": [\"RandomCondition\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Condition (must have operator StringEquals or StringLike and key name " +
+ "s3:prefix) - [\"RandomCondition\"]", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidConditionAttributeMissingStringEqualsThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": null }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Missing Condition operator value for StringEquals",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidConditionAttributeStructureThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": [{ \"s3:prefix\": \"folder/\" }] }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Condition operator value structure - [{\"s3:prefix\":\"folder/\"}]",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidJsonThrows() {
+ final String invalidJson = "{[{{}]\"\"";
+
+ expectResolveThrowsForBothAuthorizers(
+ invalidJson, "IAM session policy: Invalid policy JSON (most likely JSON structure is incorrect)",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testJsonExceedsMaxLengthThrows() {
+ final String json = createJsonStringLargerThan2048Characters();
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid policy JSON - exceeds maximum length of 2048 characters",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testJsonAtMaxLengthSucceeds() throws OMException {
+ // Create a JSON string that is exactly 2048 characters
+ final String json = create2048CharJsonString();
+ assertThat(json.length()).isEqualTo(2048);
+
+ // Must not throw an exception
+ resolve(json, VOLUME, NATIVE);
+ resolve(json, VOLUME, RANGER);
+ }
+
+ @Test
+ public void testConditionKeyMustBeCaseInsensitive() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"S3:PRefiX\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ // Must not throw exception
+ resolve(json, VOLUME, NATIVE);
+ resolve(json, VOLUME, RANGER);
+ }
+
+ @Test
+ public void testEffectMustBeCaseSensitive() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"aLLOw\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Effect - aLLOw", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMapMatchesConstant() {
+ assertThat(buildCaseInsensitiveS3ActionMap()).isEqualTo(IamSessionPolicyResolver.S3_ACTION_MAP_CI);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMap() {
+ final Map<String, Set<S3Action>> caseInsensitiveS3ActionMap = buildCaseInsensitiveS3ActionMap();
+ // Verify that individual S3 actions are present
+ assertThat(caseInsensitiveS3ActionMap).containsKeys(
+ "s3:listbucket", "s3:getobject", "s3:putobject", "s3:deleteobject", "s3:createbucket", "s3:listallmybuckets");
+
+ // Verify that wildcard actions are present
+ assertThat(caseInsensitiveS3ActionMap).containsKeys(
+ "s3:*", "s3:get*", "s3:put*", "s3:list*", "s3:delete*", "s3:create*");
+
+ // Verify s3:Get* contains Get actions
+ final Set<S3Action> getActions = caseInsensitiveS3ActionMap.get("s3:get*");
+ assertThat(getActions).containsOnly(
+ S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+
+ // Verify s3:Put* contains Put actions
+ final Set<S3Action> putActions = caseInsensitiveS3ActionMap.get("s3:put*");
+ assertThat(putActions).containsOnly(
+ S3Action.PUT_OBJECT, S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL);
+
+ // Verify s3:List* contains List actions
+ final Set<S3Action> listActions = caseInsensitiveS3ActionMap.get("s3:list*");
+ assertThat(listActions).containsOnly(
+ S3Action.LIST_BUCKET, S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET_MULTIPART_UPLOADS,
+ S3Action.LIST_MULTIPART_UPLOAD_PARTS);
+
+ // Verify s3:Delete* contains Delete actions
+ final Set<S3Action> deleteActions = caseInsensitiveS3ActionMap.get("s3:delete*");
+ assertThat(deleteActions).containsOnly(
+ S3Action.DELETE_OBJECT, S3Action.DELETE_BUCKET, S3Action.DELETE_OBJECT_TAGGING);
+
+ // Verify s3:Create* contains Create actions
+ final Set<S3Action> createActions = caseInsensitiveS3ActionMap.get("s3:create*");
+ assertThat(createActions).containsOnly(S3Action.CREATE_BUCKET);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMapIndividualActionsContainSingleEntry() {
+ final Map<String, Set<S3Action>> actionMap = buildCaseInsensitiveS3ActionMap();
+
+ // Individual actions should map to a set with exactly one entry
+ final Set<S3Action> listBucketAction = actionMap.get("s3:listbucket");
+ assertThat(listBucketAction).hasSize(1);
+
+ final Set<S3Action> getObjectAction = actionMap.get("s3:getobject");
+ assertThat(getObjectAction).hasSize(1);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithNullReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(null);
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithEmptyListReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(emptySet());
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithSingleActionMapsCorrectly() {
+ final Set<S3Action> listBucket = mapPolicyActionsToS3Actions(Collections.singleton("s3:ListBucket"));
+ assertThat(listBucket).containsOnly(S3Action.LIST_BUCKET);
+
+ // Ensure case-insensitive action works
+ final Set<S3Action> listBucketCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:ListBuCKet"));
+ assertThat(listBucketCi).containsOnly(S3Action.LIST_BUCKET);
+
+ final Set<S3Action> deleteObject = mapPolicyActionsToS3Actions(Collections.singleton("s3:DeleteObject"));
+ assertThat(deleteObject).containsOnly(S3Action.DELETE_OBJECT);
+
+ // Ensure case-insensitive action works
+ final Set<S3Action> deleteObjectCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:DeLETeObjeCT"));
+ assertThat(deleteObjectCi).containsOnly(S3Action.DELETE_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithMultipleActionsMapAllCorrectly() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:ListBucket", "s3:GetObject", "s3:PutObject"));
+ assertThat(result).containsOnly(S3Action.LIST_BUCKET, S3Action.GET_OBJECT, S3Action.PUT_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithWildcardExpansion() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(Collections.singleton("s3:Get*"));
+ assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+
+ // Ensure it is case-insensitive
+ final Set<S3Action> resultCi = mapPolicyActionsToS3Actions(Collections.singleton("s3:gET*"));
+ assertThat(resultCi).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithS3StarReturnsAll() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(Collections.singleton("s3:*"));
+ assertThat(result).containsOnly(S3Action.ALL_S3);
+
+ final Set<S3Action> resultCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:*"));
+ assertThat(resultCi).containsOnly(S3Action.ALL_S3);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsIgnoresUnsupportedActions() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:GetAccelerateConfiguration", "s3:GetObject"));
+ // Unsupported action should be silently ignored
+ assertThat(result).containsOnly(S3Action.GET_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithOnlyUnsupportedActionsReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(
+ strSet("s3:GetAccelerateConfiguration", "s3:PutBucketVersioning"));
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsDeduplicatesResults() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:GetObject", "s3:GetBucketAcl"));
+ assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsHandlesMultipleWildcards() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:Put*"));
+ assertThat(result).containsOnly(
+ S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING, S3Action.PUT_OBJECT,
+ S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithS3StarIgnoresOtherActions() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:*", "s3:GetObject", "s3:PutObject"));
+ // When s3:* is present, it should return only the ALL_S3 action
+ assertThat(result).containsOnly(S3Action.ALL_S3);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithWildcard() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("*"));
+ assertThat(resultRanger).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.ANY, "*", null, null));
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithSingleBucket() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.BUCKET, "my-bucket", null, null);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::my-bucket"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::my-bucket"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcard() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.BUCKET_WILDCARD, "my-bucket*", null, null);
+
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::my-bucket*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::my-bucket*"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcardAndExactObjectKey() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "*", null, "myKey.txt");
+
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::*/myKey.txt")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::*/myKey.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcardAndObjectWildcard() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::*/*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "*", "*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::*/*"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndExactObjectKey() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt");
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket1/key.txt"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket1/key.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndExactObjectKeyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "bucket2", null, "path/folder/nested/key.txt");
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket2/path/folder/nested/key.txt"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket2/path/folder/nested/key.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndEmpty() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndEmptyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "path/b/", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/path/b/*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "path/b/*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/path/b/*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndNonEmpty() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "test", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/test*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "test*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/test*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndNonEmptyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket", "a/b/test", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket/a/b/test*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket", "a/b/test*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket/a/b/test*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardNotAtEnd() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*.log")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*.log", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*.log"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardNotAtEndWithPath() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket/a/q/*.ps")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer if " +
+ "wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket", "a/q/*.ps", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket/a/q/*.ps"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardOneAtEndAndOneNotAtEnd()
+ throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*key*")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*key*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*key*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardOneAtEndAndOneNotAtEndWithPath()
+ throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/a/b/t/*key*")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "a/b/t/*key*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/a/b/t/*key*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithMultipleResources() throws OMException {
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, strSet("arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2/*", "arn:aws:s3:::bucket3/key.txt"));
+ assertThat(resultNative).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket2", "", null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket3", null, "key.txt"));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, strSet("arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2/*", "arn:aws:s3:::bucket3/key.txt"));
+ assertThat(resultRanger).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket2", "*", null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket3", null, "key.txt"));
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithInvalidArnThrows() {
+ final String invalidArn = "arn:aws:ec2:::bucket";
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton(invalidArn)),
+ "IAM session policy: Unsupported Resource Arn - " + invalidArn, NOT_SUPPORTED_OPERATION);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, Collections.singleton(invalidArn)),
+ "IAM session policy: Unsupported Resource Arn - " + invalidArn, NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithArnWithNoBucketThrows() {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::")),
+ "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, Collections.singleton("arn:aws:s3:::")),
+ "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithNoResourcesThrows() {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, emptySet()), "IAM session policy: No Resource(s) found in policy",
+ MALFORMED_POLICY_DOCUMENT);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, emptySet()), "IAM session policy: No Resource(s) found in policy",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithResourceAny() {
+ // This also tests that acls are deduplicated across different resource types
+ final Set<S3Action> actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET, S3Action.GET_OBJECT)
+ .collect(Collectors.toSet()); // actions at volume, bucket and key levels
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.ANY, "*", null, null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type ANY not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ final Set<IOzoneObj> readAndListObjects = objSet(volume(), bucket("*")); // volume, bucket level have READ, LIST
+ final Set<IOzoneObj> readObject = objSet(key("*", "*")); // key level has READ
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObjects, acls(READ, LIST)),
+ new OzoneGrant(readObject, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketResourceThatIsListBucket() {
+ final Set<S3Action> actions = Collections.singleton(IamSessionPolicyResolver.S3Action.LIST_BUCKET);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> readAndListObject = objSet(bucket("bucket1"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), prefix("bucket1", ""));
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), key("bucket1", "*"));
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketResourceThatIsNotListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.CREATE_BUCKET);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> createObject = objSet(bucket("bucket1"));
+ final Set<IOzoneObj> readObject = objSet(volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketWildcardResource() {
+ final Set<S3Action> actions = Collections.singleton(IamSessionPolicyResolver.S3Action.PUT_BUCKET_ACL);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "bucket1*", null, null));
+ final Set<IOzoneObj> readReadAclAndWriteAclObject = objSet(bucket("bucket1*"));
+ final Set<IOzoneObj> readVolume = objSet(volume());
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readReadAclAndWriteAclObject, acls(READ, READ_ACL, WRITE_ACL)),
+ new OzoneGrant(readVolume, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketsWildcardResourceAll() {
+ // For AWS IAM, s3:ListAllMyBuckets supports both "*" and "arn:aws:s3:::*" as
+ // Resource values. The "*" case is covered by testCreatePathsAndPermissionsWithResourceAny.
+ // This test ensures that "arn:aws:s3:::*" (parsed as BUCKET_WILDCARD with bucket="*")
+ // also grants the expected volume-level permissions for ListAllMyBuckets.
+ final Set<S3Action> actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "*", null, null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+
+ // Both the volume and the wildcard bucket should end up with READ + LIST permissions.
+ // We also need READ access on the keys
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ final Set<IOzoneObj> readAndListObjects = objSet(volume(), bucket("*"));
+ final Set<IOzoneObj> readObjects = objSet(key("*", "*"));
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObjects, acls(READ, LIST)), new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectExactResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readObjects = objSet(key("bucket1", "key.txt"), bucket("bucket1"), volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(readObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithDeleteObjectGrantsDeleteOnKey() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.DELETE_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readVolumeAndBucket = objSet(volume(), bucket("bucket1"));
+ final Set<IOzoneObj> deleteKey = objSet(key("bucket1", "key.txt"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithAbortMultipartUploadGrantsWriteOnKey() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.ABORT_MULTIPART_UPLOAD);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readVolumeAndBucket = objSet(volume(), bucket("bucket1"));
+ final Set<IOzoneObj> writeKey = objSet(key("bucket1", "key.txt"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectPrefixResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "prefix/", null));
+ final Set<IOzoneObj> nativeReadObjects = objSet(prefix("bucket1", "prefix/"), bucket("bucket1"), volume());
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type OBJECT_PREFIX not supported for RangerOzoneAuthorizer");
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectPrefixWildcardResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "prefix/*", null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type OBJECT_PREFIX_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Set<IOzoneObj> rangerReadObjects = objSet(key("bucket1", "prefix/*"), bucket("bucket1"), volume());
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForObjectActionMustIgnoreConditionPrefixes() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "", null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ final Set<IOzoneObj> nativeReadObjects = objSet(prefix("bucket1", ""), bucket("bucket1"), volume());
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "*", null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ final Set<IOzoneObj> rangerReadObjects = objSet(key("bucket1", "*"), bucket("bucket1"), volume());
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.LIST_BUCKET);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> nativeReadObjects = objSet(
+ prefix("bucket1", "folder1/"), prefix("bucket1", "folder2/"), volume());
+ final Set<IOzoneObj> nativeReadAndListObject = objSet(bucket("bucket1"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(nativeReadObjects, acls(READ)), new OzoneGrant(nativeReadAndListObject, acls(READ, LIST)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> rangerReadObjects = objSet(
+ key("bucket1", "folder1/"), key("bucket1", "folder2/"), volume());
+ final Set<IOzoneObj> rangerReadAndListObject = objSet(bucket("bucket1"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(rangerReadObjects, acls(READ)), new OzoneGrant(rangerReadAndListObject, acls(READ, LIST)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsNotListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_BUCKET_ACL);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+ final Set<IOzoneObj> readObject = objSet(volume());
+ final Set<IOzoneObj> readAndReadAclObject = objSet(bucket("bucket1"));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithNoMappedActions() {
+ final Set<S3Action> actions = emptySet();
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).isEmpty();
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).isEmpty();
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithNoMappedResources() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = emptySet();
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).isEmpty();
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).isEmpty();
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsDeduplicatesAcrossSameResourceTypes() {
+ final Set<S3Action> actions = Stream.of(
+ S3Action.GET_OBJECT, S3Action.GET_OBJECT_TAGGING, S3Action.DELETE_OBJECT, S3Action.DELETE_OBJECT_TAGGING)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readAndDeleteAndWriteObject = objSet(key("bucket1", "key.txt"));
+ final Set<IOzoneObj> readObjects = objSet(bucket("bucket1"), volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)),
+ new OzoneGrant(readObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)),
+ new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithAllS3ActionsOverridesAnyOtherAction() {
+ final Set<S3Action> actions = Stream.of(
+ S3Action.ALL_S3, S3Action.GET_OBJECT, S3Action.DELETE_OBJECT, S3Action.LIST_BUCKET)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Stream.of(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket2", null, null))
+ .collect(Collectors.toSet());
+ final Set<IOzoneObj> allObjects = objSet(key("bucket1", "key.txt"), bucket("bucket2"));
+
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), bucket("bucket1"), prefix("bucket2", ""));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), bucket("bucket1"), key("bucket2", "*"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsWhenSameStatementsArePresent() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsForSameActionsButDifferentResource() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket2\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"), bucket("my-bucket2"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(
+ objSet(volume(), prefix("my-bucket2", ""), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(
+ objSet(volume(), key("my-bucket2", "*"), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsForDifferentActionsButSameResource() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:CreateBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL, CREATE);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsWhenAllActionPresent() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket ALL (instead of individual actions); volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket ALL (instead of individual actions); volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllowGetPutOnKey() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\"s3:GetObject\", \"s3:PutObject\"],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/folder/file.txt\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedFromBothAuthorizers = new LinkedHashSet<>();
+ // Expected: READ, CREATE, WRITE on key; bucket READ; volume READ
+ final Set<IOzoneObj> keySet = objSet(key("my-bucket", "folder/file.txt"));
+ final Set<ACLType> keyAcls = acls(READ, CREATE, WRITE);
+ expectedResolvedFromBothAuthorizers.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ expectedResolvedFromBothAuthorizers.add(new OzoneGrant(keySet, keyAcls));
+
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers);
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers);
+ }
+
+ @Test
+ public void testAllActionsForKey() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all key ACLs on prefix "" under bucket; bucket READ, volume READ
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> allKeyAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, allKeyAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: all key acls for resource type KEY with key name "*"
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, allKeyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsForKeyWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected - only ListBucket supports s3:prefix and that is a bucket action,
+ // not object action
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testAllActionsForBucket() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all Bucket ACLs for bucket; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> allBucketAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, allBucketAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: all Bucket ACLs for bucket; volume, key "*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, allBucketAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsForBucketWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ;
+ // prefix "team/folder", "team/folder/" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(
+ new OzoneGrant(objSet(volume(), prefix("my-bucket", "team/folder"), prefix("my-bucket", "team/folder/")),
+ acls(READ)));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ,
+ // key "team/folder", "team/folder/*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(
+ new OzoneGrant(objSet(volume(), key("my-bucket", "team/folder"), key("my-bucket", "team/folder/*")),
+ acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesInSeparateStatements() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ // Expected for native: all key ACLs on prefix "" under bucket
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> keyAllAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ // Expected for Ranger: all key acls for resource type KEY with key name "*"
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesInOneStatement() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:*\"\n" +
+ " ],\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all for bucket and key acls; volume READ
+ final Set<IOzoneObj> resourceSetNative = objSet(bucket("my-bucket"), prefix("my-bucket", ""));
+ expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(ALL)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: all for bucket and key acls; volume READ
+ final Set<IOzoneObj> resourceSetRanger = objSet(bucket("my-bucket"), key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(ALL)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesWithDifferentBucketsAndDeepPathsInOneStatement() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:*\"\n" +
+ " ],\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket/team/folder1/security/*\",\n" +
+ " \"arn:aws:s3:::my-bucket2/team/folder2/misc/*\"\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all key ACLs on prefix "team/folder1/security/" under
+ // my-bucket and all key ACLs on prefix "team/folder2/misc/" under my-bucket2; bucket READ; volume READ
+ final Set<IOzoneObj> keyPrefixSet = objSet(
+ prefix("my-bucket", "team/folder1/security/"), prefix("my-bucket2", "team/folder2/misc/"));
+ final Set<ACLType> keyAllAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: all key acls for resource type KEY with key name
+ // "team/folder1/security/*" under my-bucket and "team/folder2/misc/*" under my-bucket2; bucket READ; volume READ
+ final Set<IOzoneObj> rangerKeySet = objSet(
+ key("my-bucket", "team/folder1/security/*"), key("my-bucket2", "team/folder2/misc/*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testUnsupportedActionIgnoredWhenItIsTheOnlyAction() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ReplicateObject\",\n" + // unsupported action
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testUnsupportedResourceArnThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:dynamodb:us-east-2:123456789012:table/example-table\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Resource Arn - " +
+ "arn:aws:dynamodb:us-east-2:123456789012:table/example-table", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testListBucketWithWildcard() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::proj-*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ and LIST on wildcard pattern; volume and key "*" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("proj-*"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("proj-*", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testListBucketOperationsWithNoPrefixes() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:ListBucket\",\n" +
+ " \"s3:ListBucketMultipartUploads\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::proj\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ and LIST; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("proj"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), prefix("proj", ""));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(nativeReadObjects, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: bucket READ and LIST; volume, key "*" READ
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), key("proj", "*"));
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(rangerReadObjects, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testIgnoresUnsupportedActionsWhenSupportedActionsAreIncluded() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Sid\": \"AllowListingOfDataLakeFolder\",\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetAccelerateConfiguration\",\n" + // unsupported action
+ " \"s3:GetBucketAcl\",\n" + // ignored because it doesn't support s3:prefix condition
+ " \"s3:GetObject\",\n" + // object-level action not applied for bucket
+ " \"s3:GetObjectAcl\",\n" + // unsupported action
+ " \"s3:ListBucket\",\n" +
+ " \"s3:ListBucketMultipartUploads\"\n" + // ignored because it doesn't support s3:prefix condition
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+
+ // Expected for native: READ, LIST bucket acls; volume and prefixes "team/folder", "team/folder/" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("bucket1"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(
+ objSet(volume(), prefix("bucket1", "team/folder"), prefix("bucket1", "team/folder/")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST bucket acls; volume and keys "team/folder" and "team/folder/*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(
+ objSet(volume(), key("bucket1", "team/folder"), key("bucket1", "team/folder/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultiplePrefixesWithWildcards() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/*\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": [\"a/*\", \"b/*\"] } }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // s3:prefix conditions do not apply to object actions like s3:GetObject.
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testListAndGetWithPrefixConditionSkipsObjectAction() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\"s3:ListBucket\", \"s3:GetObject\"],\n" +
+ " \"Resource\": [\"arn:aws:s3:::logs\", \"arn:aws:s3:::logs/*\"],\n" +
+ " \"Condition\": { \"StringLike\": { \"s3:prefix\": \"team/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Expected for native (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
+ // prefix "log/team" READ
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ expectedResolvedNative.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("logs", "team/")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
+ // key "log/team/*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("logs", "team/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testListBucketOnObjectResourceReturnsEmpty() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testObjectResourceWithWildcardInMiddle() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/file*.log\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards in middle of object resource are not supported for Native authorizer
+ expectResolveThrows(
+ json, NATIVE, "IAM session policy: Wildcard prefix patterns are not supported for Ozone native " +
+ "authorizer if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ acl on key "file*.log", bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("logs", "file*.log"), bucket("logs"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testObjectResourceWithPrefixWildcard() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::myBucket/file*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ acl on prefix "file" under bucket, bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("myBucket", "file"), bucket("myBucket"), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ acl on key "file*", bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("myBucket", "file*"), bucket("myBucket"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testBucketActionOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:ListAllMyBuckets\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ and LIST on volume and bucket (wildcard), READ on key "*"
+ final Set<IOzoneObj> resourceSet = objSet(volume(), bucket("*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(READ, LIST)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testObjectActionOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:PutObject\",\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: CREATE and WRITE key acls on wildcard pattern, bucket READ, volume READ
+ final Set<IOzoneObj> keySet = objSet(key("*", "*"));
+ final Set<ACLType> keyAcls = acls(CREATE, WRITE);
+ expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllResourcesWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"*\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: (only ListBucket supports s3:prefix) READ volume; READ, LIST acl on bucket;
+ // READ on key "team/folder", "team/folder/*"
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST)));
+ expectedResolvedRanger.add(
+ new OzoneGrant(objSet(volume(), key("*", "team/folder"), key("*", "team/folder/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST acl on volume, ALL acl bucket (wildcard) and key (wildcard)
+ final Set<IOzoneObj> resourceSet = objSet(bucket("*"), key("*", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(ALL)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllBucketResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: ALL bucket acls on wildcard pattern, volume READ, key "*" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("*"));
+ final Set<ACLType> bucketAcls = acls(ALL);
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllObjectResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::*/*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: ALL key acls on wildcard pattern; bucket READ; volume READ
+ final Set<IOzoneObj> keySet = objSet(key("*", "*"));
+ final Set<ACLType> keyAcls = acls(ALL);
+ expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupGetStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Get*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, READ_ACL acls
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, READ_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for native: READ acl on prefix "" under bucket; volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("my-bucket", ""), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, READ_ACL acls
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("my-bucket", "*"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupListStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:List*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" + // ListMultipartUploadParts has READ effect on file/object resources
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ, LIST bucket acls
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for native: READ acl on prefix "" under bucket; volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("my-bucket", ""), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST bucket acls
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("my-bucket", "*"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupPutStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Put*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, READ_ACL, WRITE_ACL acl
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcl = acls(READ, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcl));
+ // Expected for native: CREATE, WRITE acls on prefix "" under bucket
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> keyAcls = acls(CREATE, WRITE);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAcls));
+ // Expected for native: volume READ
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, READ_ACL, WRITE_ACL acl
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcl));
+ // Expected for Ranger: CREATE, WRITE key acls for resource type KEY with key name "*"
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAcls));
+ // Expected for Ranger: volume READ
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupDeleteStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Delete*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: DELETE and WRITE on prefix "" under bucket; bucket READ, DELETE; volume READ
+ final Set<IOzoneObj> resourceSetNative = objSet(prefix("my-bucket", ""));
+ expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(DELETE, WRITE)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: DELETE and WRITE on resource type KEY with key name "*"; bucket READ, DELETE; volume READ
+ final Set<IOzoneObj> resourceSetRanger = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(DELETE, WRITE)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMismatchedActionAndResourceReturnsEmpty() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" + // object-level action
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" + // bucket-level resource
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testInvalidResourceArnThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ private static void expectIllegalArgumentException(Runnable runnable, String expectedMessage) {
+ try {
+ runnable.run();
+ throw new AssertionError("Expected exception not thrown");
+ } catch (IllegalArgumentException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ }
+ }
+
+ private static void expectOMExceptionWithCode(RunnableThrowingOMException runnable, String expectedMessage,
+ OMException.ResultCodes expectedCode) {
+ try {
+ runnable.run();
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ assertThat(ex.getResult()).isEqualTo(expectedCode);
+ }
+ }
+
+ @FunctionalInterface
+ private interface RunnableThrowingOMException {
+ void run() throws OMException;
+ }
+
+ private static IOzoneObj key(String bucket, String key) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.KEY)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .build();
+ }
+
+ private static IOzoneObj volume() {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.VOLUME)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .build();
+ }
+
+ private static IOzoneObj bucket(String bucket) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.BUCKET)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .build();
+ }
+
+ private static IOzoneObj prefix(String bucket, String prefix) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.PREFIX)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .setPrefixName(prefix)
+ .build();
+ }
+
+ private static Set<IOzoneObj> objSet(IOzoneObj... objs) {
+ final Set<IOzoneObj> s = new LinkedHashSet<>();
+ Collections.addAll(s, objs);
+ return s;
+ }
+
+ private static Set<ACLType> acls(ACLType... types) {
+ final Set<ACLType> s = new LinkedHashSet<>();
+ Collections.addAll(s, types);
+ return s;
+ }
+
+ private static Set<String> strSet(String... strs) {
+ final Set<String> s = new LinkedHashSet<>();
+ Collections.addAll(s, strs);
+ return s;
+ }
+
+ private static void expectResolveThrows(String json, IamSessionPolicyResolver.AuthorizerType authorizerType,
+ String expectedMessage, OMException.ResultCodes expectedCode) {
+ try {
+ resolve(json, VOLUME, authorizerType);
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ assertThat(ex.getResult()).isEqualTo(expectedCode);
+ }
+ }
+
+ private static void expectResolveThrowsForBothAuthorizers(String json, String expectedMessage,
+ OMException.ResultCodes expectedCode) {
+ expectResolveThrows(json, NATIVE, expectedMessage, expectedCode);
+ expectResolveThrows(json, RANGER, expectedMessage, expectedCode);
+ }
+
+ /**
+ * Ensure resources containing wildcards in buckets throw an Exception
+ * when the OzoneNativeAuthorizer is used.
+ */
+ private static void expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(String json) {
+ try {
+ resolve(json, VOLUME, NATIVE);
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer");
+ assertThat(ex.getResult()).isEqualTo(NOT_SUPPORTED_OPERATION);
+ }
+ }
+
+ private static String createJsonStringLargerThan2048Characters() {
+ final StringBuilder jsonBuilder = new StringBuilder();
- jsonBuilder.append("{\n");
- jsonBuilder.append(" \"Statement\": [{\n");
- jsonBuilder.append(" \"Effect\": \"Allow\",\n");
- jsonBuilder.append(" \"Action\": \"s3:ListBucket\",\n");
- jsonBuilder.append(" \"Resource\": \"arn:aws:s3:::");
++ jsonBuilder.append("{\n")
++ .append(" \"Statement\": [{\n")
++ .append(" \"Effect\": \"Allow\",\n")
++ .append(" \"Action\": \"s3:ListBucket\",\n")
++ .append(" \"Resource\": \"arn:aws:s3:::");
+ // Add enough characters to exceed 2048
+ while (jsonBuilder.length() < 2048) {
+ jsonBuilder.append("very-long-bucket-name-that-exceeds-the-limit-");
+ }
- jsonBuilder.append("\"\n");
- jsonBuilder.append(" }]\n");
- jsonBuilder.append('}');
++ jsonBuilder.append("\"\n")
++ .append(" }]\n")
++ .append('}');
+ return jsonBuilder.toString();
+ }
+
+ private static String create2048CharJsonString() {
+ final StringBuilder jsonBuilder = new StringBuilder();
- jsonBuilder.append("{\n");
- jsonBuilder.append(" \"Statement\": [{\n");
- jsonBuilder.append(" \"Effect\": \"Allow\",\n");
- jsonBuilder.append(" \"Action\": \"s3:ListBucket\",\n");
- jsonBuilder.append(" \"Resource\": \"arn:aws:s3:::");
++ jsonBuilder.append("{\n")
++ .append(" \"Statement\": [{\n")
++ .append(" \"Effect\": \"Allow\",\n")
++ .append(" \"Action\": \"s3:ListBucket\",\n")
++ .append(" \"Resource\": \"arn:aws:s3:::");
+ // Add characters to reach exactly 2048 (accounting for closing brackets and newlines)
+ // Closing part: "\"\n }]\n}" = 8 chars
+ while (jsonBuilder.length() < 2048 - 8) {
+ jsonBuilder.append('a');
+ }
+ jsonBuilder.append("\"\n }]\n}");
+ return jsonBuilder.toString();
+ }
+}
+
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index edb1484fd0f,d4a340b311a..f9078360799
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -46,10 -43,6 +44,7 @@@ public class TestOzoneConfigurationFiel
ReconConfigKeys.class, ReconServerConfigKeys.class,
S3GatewayConfigKeys.class,
S3SecretConfigKeys.class,
- S3STSConfigKeys.class,
- SCMHTTPServerConfig.class,
- SCMHTTPServerConfig.ConfigStrings.class,
- ScmConfig.ConfigStrings.class
++ S3STSConfigKeys.class
};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 9bb0d801ee7,22c71cc2985..dde32437db5
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -156,9 -156,7 +156,10 @@@ enum Type
PutObjectTagging = 140;
GetObjectTagging = 141;
DeleteObjectTagging = 142;
- AssumeRole = 143;
- RevokeSTSToken = 144;
- DeleteRevokedSTSTokens = 145;
+ SubmitSnapshotDiff = 143;
++ AssumeRole = 144;
++ RevokeSTSToken = 145;
++ DeleteRevokedSTSTokens = 146;
}
enum SafeMode {
@@@ -309,10 -307,8 +310,11 @@@ message OMRequest
optional PutObjectTaggingRequest putObjectTaggingRequest = 141;
optional DeleteObjectTaggingRequest deleteObjectTaggingRequest = 142;
repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 143;
- optional AssumeRoleRequest assumeRoleRequest = 144;
- optional RevokeSTSTokenRequest revokeSTSTokenRequest = 145;
- optional DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = 146;
- optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 147;
-
- optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
++ optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
++ optional AssumeRoleRequest assumeRoleRequest = 145;
++ optional RevokeSTSTokenRequest revokeSTSTokenRequest = 146;
++ optional DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = 147;
++ optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 148;
}
message OMResponse {
@@@ -443,12 -439,11 +445,13 @@@
optional GetQuotaRepairStatusResponse GetQuotaRepairStatusResponse = 136;
optional StartQuotaRepairResponse StartQuotaRepairResponse = 137;
- optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
- optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
- optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
-
- optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
+ optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
+ optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
+ optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
- optional AssumeRoleResponse assumeRoleResponse = 143;
- optional RevokeSTSTokenResponse revokeSTSTokenResponse = 144;
- optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 145;
++ optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
++ optional AssumeRoleResponse assumeRoleResponse = 144;
++ optional RevokeSTSTokenResponse revokeSTSTokenResponse = 145;
++ optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 146;
}
enum Status {
@@@ -579,8 -574,11 +582,13 @@@
TOO_MANY_SNAPSHOTS = 98;
- REVOKED_TOKEN = 99;
- MALFORMED_POLICY_DOCUMENT = 100;
+ ETAG_MISMATCH = 99;
+
+ ETAG_NOT_AVAILABLE = 100;
+
+ ATOMIC_WRITE_CONFLICT = 101;
++ REVOKED_TOKEN = 102;
++ MALFORMED_POLICY_DOCUMENT = 103;
}
/**
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index b14f01cf6ba,a5ba074156e..46573917c88
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@@ -645,26 -596,12 +645,26 @@@ public class OmMetadataReader implement
"Bucket:" + obj.getBucketName() + " " : "";
String keyName = obj.getKeyName() != null ?
"Key:" + obj.getKeyName() : "";
+ // For STS tokens, make clear that the user is using an assumed role, otherwise the access denied
+ // message could be confusing
+ String user = normalizedRequestContext.getClientUgi().getShortUserName();
+ final STSTokenIdentifier stsTokenIdentifier = OzoneManager.getStsTokenIdentifier();
+ if (stsTokenIdentifier != null) {
- final StringBuilder builder = new StringBuilder(user);
- builder.append(" (STS assumed role arn = ");
- builder.append(stsTokenIdentifier.getRoleArn());
- builder.append(", tempAccessKeyId = ");
- builder.append(stsTokenIdentifier.getTempAccessKeyId());
- builder.append(')');
++ final StringBuilder builder = new StringBuilder(user)
++ .append(" (STS assumed role arn = ")
++ .append(stsTokenIdentifier.getRoleArn())
++ .append(", tempAccessKeyId = ")
++ .append(stsTokenIdentifier.getTempAccessKeyId())
++ .append(')');
+ user = builder.toString();
+ }
log.warn("User {} doesn't have {} permission to access {} {}{}{}",
- context.getClientUgi().getShortUserName(), context.getAclRights(),
+ user,
+ normalizedRequestContext.getAclRights(),
obj.getResourceType(), volumeName, bucketName, keyName);
throw new OMException(
- "User " + context.getClientUgi().getShortUserName() +
- " doesn't have " + context.getAclRights() +
+ "User " + user +
+ " doesn't have " + normalizedRequestContext.getAclRights() +
" permission to access " + obj.getResourceType() + " " +
volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d1f27c6132c,d258324b3bb..1986b34acb3
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -684,12 -695,7 +704,13 @@@ public final class OzoneManager extend
this.isS3MultiTenancyEnabled =
OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf);
+ // Enable S3 STS if config key is set
+ this.isS3STSEnabled = conf.getBoolean(
+ OzoneConfigKeys.OZONE_S3G_STS_HTTP_ENABLED_KEY,
+ OzoneConfigKeys.OZONE_S3G_STS_HTTP_ENABLED_DEFAULT);
+
- metrics = OMMetrics.create();
+ metrics = OMMetrics.create(conf);
++
omSnapshotIntMetrics = OmSnapshotInternalMetrics.create();
perfMetrics = OMPerformanceMetrics.register();
omDeletionMetrics = DeletingServiceMetrics.create();
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
index 1ba4b7186f2,00000000000..8c13aac5190
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
@@@ -1,285 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.UUID;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.security.token.ShortLivedTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+
+/**
+ * Token identifier for STS (Security Token Service) tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class STSTokenIdentifier extends ShortLivedTokenIdentifier {
+ public static final Text KIND_NAME = new Text("STSToken");
+
+ // STS-specific fields
+ private String roleArn;
+ private String originalAccessKeyId;
+ private String secretAccessKey;
+ private String sessionPolicy;
+
+ // Encryption key derived from ManagedSecretKey for this token
+ private transient byte[] encryptionKey;
+
+ // Service name for STS tokens
+ public static final String STS_SERVICE = "STS";
+
+ /**
+ * Create an empty STS token identifier.
+ */
+ public STSTokenIdentifier() {
+ super();
+ }
+
+ /**
+ * Create a new STS token identifier with encryption support.
+ *
+ * @param tempAccessKeyId the temporary access key ID (owner)
+ * @param originalAccessKeyId the original long-lived access key ID that created this token
+ * @param roleArn the ARN of the assumed role
+ * @param expiry the token expiration time
+ * @param secretAccessKey the secret access key associated with the temporary access key ID
+ * @param sessionPolicy an optional opaque identifier that further limits the scope of
+ * the permissions granted by the role
+ * @param encryptionKey the key bytes for encrypting sensitive fields
+ */
+ public STSTokenIdentifier(String tempAccessKeyId, String originalAccessKeyId, String roleArn, Instant expiry,
+ String secretAccessKey, String sessionPolicy, byte[] encryptionKey) {
+ super(tempAccessKeyId, expiry);
+ this.originalAccessKeyId = originalAccessKeyId;
+ this.roleArn = roleArn;
+ this.secretAccessKey = secretAccessKey;
+ this.sessionPolicy = sessionPolicy;
+ this.encryptionKey = encryptionKey != null ? encryptionKey.clone() : null;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String getService() {
+ return STS_SERVICE;
+ }
+
+ @Override
+ public void readFromByteArray(byte[] bytes) throws IOException {
+ final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
+ readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.write(toProtoBuf().toByteArray());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ fromProtoBuf(token);
+ }
+
+ /**
+ * Convert this identifier to protobuf format.
+ */
+ public OMTokenProto toProtoBuf() {
+ Preconditions.checkArgument(this.encryptionKey != null, "The encryption key must not be null");
+
+ final OMTokenProto.Builder builder = OMTokenProto.newBuilder();
+ // Note: secretKeyId must be set before attempting to decrypt secretAccessKey
+ if (getSecretKeyId() != null) {
+ builder.setSecretKeyId(getSecretKeyId().toString());
+ }
+
+ builder
+ .setType(OMTokenProto.Type.S3_STS_TOKEN)
+ .setMaxDate(getExpiry().toEpochMilli())
+ .setOwner(getOwnerId() != null ? getOwnerId() : "")
+ .setAccessKeyId(getOwnerId() != null ? getOwnerId() : "")
+ .setOriginalAccessKeyId(originalAccessKeyId != null ? originalAccessKeyId : "")
+ .setRoleArn(roleArn != null ? roleArn : "")
+ .setSecretAccessKey(secretAccessKey != null ? encryptSensitiveField(secretAccessKey) : "")
+ .setSessionPolicy(sessionPolicy != null ? sessionPolicy : "");
+
+ return builder.build();
+ }
+
+ /**
+ * Initialize this identifier from protobuf.
+ */
+ public void fromProtoBuf(OMTokenProto token) throws IOException {
+ Preconditions.checkArgument(
+ token.getType() == OMTokenProto.Type.S3_STS_TOKEN,
+ "Invalid token type for STSTokenIdentifier: " + token.getType());
+ Preconditions.checkArgument(this.encryptionKey != null, "The encryption key must not be null");
+
+ setOwnerId(token.getOwner());
+ setExpiry(Instant.ofEpochMilli(token.getMaxDate()));
+
+ if (token.hasOriginalAccessKeyId()) {
+ this.originalAccessKeyId = token.getOriginalAccessKeyId();
+ }
+ if (token.hasRoleArn()) {
+ this.roleArn = token.getRoleArn();
+ }
+ if (token.hasSecretKeyId()) {
+ try {
+ setSecretKeyId(UUID.fromString(token.getSecretKeyId()));
+ } catch (IllegalArgumentException e) {
+ // Handle invalid UUID format gracefully
+ throw new IOException(
+ "Invalid secretKeyId format in STS token: " + token.getSecretKeyId(), e);
+ }
+ }
+ // Note: secretKeyId must be set before attempting to decrypt secretAccessKey
+ if (token.hasSecretAccessKey()) {
+ this.secretAccessKey = decryptSensitiveField(token.getSecretAccessKey());
+ }
+
+ if (token.hasSessionPolicy()) {
+ this.sessionPolicy = token.getSessionPolicy();
+ }
+ }
+
+ /**
+ * Encrypt a sensitive field using the configured encryption key.
+ */
+ private String encryptSensitiveField(String value) {
+ if (encryptionKey == null) {
+ throw new IllegalStateException("Encryption key must be set before encrypting sensitive fields");
+ }
+
+ try {
+ final byte[] aad = computeAadBytes();
+ return STSTokenEncryption.encrypt(value, encryptionKey, aad);
+ } catch (STSTokenEncryption.STSTokenEncryptionException e) {
+ throw new RuntimeException("Token encryption failed", e);
+ }
+ }
+
+ /**
+ * Decrypt a sensitive field using the configured encryption key.
+ */
+ private String decryptSensitiveField(String encryptedValue) {
+ if (encryptionKey == null) {
+ throw new IllegalStateException("Encryption key must be set before decrypting sensitive fields");
+ }
+
+ try {
+ final byte[] aad = computeAadBytes();
+ return STSTokenEncryption.decrypt(encryptedValue, encryptionKey, aad);
+ } catch (STSTokenEncryption.STSTokenEncryptionException e) {
+ throw new RuntimeException("Token decryption failed", e);
+ }
+ }
+
+ /**
+ * Compute additional authenticated data to bind token context to encryption.
+ * Includes token type, ownerId, expiry millis, and secretKeyId.
+ */
+ private byte[] computeAadBytes() {
- final StringBuilder stringBuilder = new StringBuilder("v1|S3_STS_TOKEN|");
- stringBuilder.append(getOwnerId());
- stringBuilder.append('|');
- stringBuilder.append(getExpiry().toEpochMilli());
- stringBuilder.append('|');
- stringBuilder.append(getSecretKeyId().toString());
++ final StringBuilder stringBuilder = new StringBuilder("v1|S3_STS_TOKEN|")
++ .append(getOwnerId())
++ .append('|')
++ .append(getExpiry().toEpochMilli())
++ .append('|')
++ .append(getSecretKeyId().toString());
+ final String aad = stringBuilder.toString();
+ return aad.getBytes(StandardCharsets.UTF_8);
+ }
+
+ public String getRoleArn() {
+ return roleArn;
+ }
+
+ public String getSecretAccessKey() {
+ return secretAccessKey;
+ }
+
+ public String getOriginalAccessKeyId() {
+ return originalAccessKeyId;
+ }
+
+ /**
+ * Get the temporary access key ID (same as owner).
+ */
+ public String getTempAccessKeyId() {
+ return getOwnerId();
+ }
+
+ /**
+ * Optional session policy associated with this STS token, or null/empty if none.
+ */
+ public String getSessionPolicy() {
+ return sessionPolicy;
+ }
+
+ public void setEncryptionKey(byte[] encryptionKey) {
+ this.encryptionKey = encryptionKey.clone();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ if (!super.equals(o)) {
+ return false;
+ }
+
+ final STSTokenIdentifier that = (STSTokenIdentifier) o;
+ return Objects.equals(roleArn, that.roleArn) && Objects.equals(secretAccessKey, that.secretAccessKey) &&
+ Objects.equals(originalAccessKeyId, that.originalAccessKeyId) &&
+ Objects.equals(sessionPolicy, that.sessionPolicy);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ super.hashCode(), roleArn, secretAccessKey, originalAccessKeyId, sessionPolicy);
+ }
+
+ @Override
+ public String toString() {
+ // Intentionally left off secretAccessKey
+ return "STSTokenIdentifier{" + "tempAccessKeyId='" + getOwnerId() + "'" +
+ ", originalAccessKeyId='" + originalAccessKeyId + "', roleArn='" + roleArn + "'" +
+ ", expiry='" + getExpiry() + "', secretKeyId='" + getSecretKeyId() + "'" +
+ ", sessionPolicy='" + sessionPolicy + "'}";
+ }
+}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 799af5d7fa9,2fd6dee16db..1e78813e971
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@@ -161,12 -140,15 +140,10 @@@ public class BucketEndpoint extends Buc
ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow);
} catch (OMException ex) {
- auditReadFailure(s3GAction, ex);
- getMetrics().updateGetBucketFailureStats(startNanos);
+ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
- if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
- // File not found, continue and send normal response with 0 keyCount
- LOG.debug("Key Not found prefix: {}", prefix);
- } else {
- throw ex;
- }
+ handleOMException(ex, bucketName, prefix);
} catch (Exception ex) {
- getMetrics().updateGetBucketFailureStats(startNanos);
- auditReadFailure(s3GAction, ex);
+ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
throw ex;
}
@@@ -247,24 -215,17 +217,14 @@@
addKey(response, next);
count++;
}
- } else {
- addKey(response, next);
- count++;
- }
- if (count == maxKeys) {
- lastKey = next.getName();
- break;
+ if (count == maxKeys) {
+ lastKey = next.getName();
+ break;
+ }
}
+ } catch (RuntimeException ex) {
- getMetrics().updateGetBucketFailureStats(startNanos);
- auditReadFailure(s3GAction, ex);
- if (ex.getCause() instanceof OMException) {
- final OMException omException = (OMException) ex.getCause();
- if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) {
- throw ex;
- }
- handleOMException(omException, bucketName, prefix);
- } else {
- throw ex;
- }
++ handleRuntimeException(ex, context, bucketName, prefix);
}
}
@@@ -517,26 -418,12 +417,40 @@@
OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT);
// initialize handlers
- handlers = new ArrayList<>();
- addHandler(new BucketAclHandler());
- addHandler(new BucketCrudHandler());
- }
-
- private void addHandler(BucketOperationHandler handler) {
- copyDependenciesTo(handler);
- handlers.add(handler);
+ BucketOperationHandler chain = BucketOperationHandlerChain.newBuilder(this)
+ .add(new BucketAclHandler())
+ .add(new ListMultipartUploadsHandler())
+ .add(new BucketCrudHandler())
+ .add(this)
+ .build();
+ handler = new AuditingBucketOperationHandler(chain);
}
+
+ private void handleOMException(OMException ex, String bucketName, String prefix) throws OMException {
+ if (isExpiredToken(ex)) {
+ throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, ex);
+ } else if (isAccessDenied(ex)) {
+ throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
+ } else if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
+ // File not found, continue and send normal response with 0 keyCount
+ LOG.debug("Key Not found prefix: {}", prefix);
+ } else {
+ throw ex;
+ }
+ }
++
++ private void handleRuntimeException(RuntimeException ex, S3RequestContext context, String bucketName, String prefix)
++ throws OMException {
++ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
++ if (ex.getCause() instanceof OMException) {
++ final OMException omException = (OMException) ex.getCause();
++ if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) {
++ throw ex;
++ }
++ handleOMException(omException, bucketName, prefix);
++ } else {
++ throw ex;
++ }
++ }
++
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 20ad21e23f1,649b14b49cd..0d6fbe1657b
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@@ -697,15 -569,10 +584,15 @@@ public abstract class EndpointBase
AUDIT.logReadFailure(auditMessageForFailure(action, ex).build());
}
- protected boolean isAccessDenied(OMException ex) {
+ protected static boolean isAccessDenied(OMException ex) {
ResultCodes result = ex.getResult();
return result == ResultCodes.PERMISSION_DENIED
- || result == ResultCodes.INVALID_TOKEN;
+ || result == ResultCodes.INVALID_TOKEN
+ || result == ResultCodes.REVOKED_TOKEN;
+ }
+
+ protected boolean isExpiredToken(OMException ex) {
+ return ex.getResult() == ResultCodes.TOKEN_EXPIRED;
}
protected ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket) throws OS3Exception {
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index d97c514f9ae,2021790b332..dd09bff5a41
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@@ -558,14 -561,8 +561,14 @@@ public class ObjectEndpoint extends Obj
if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
// Just return 404 with no content
return Response.status(Status.NOT_FOUND).build();
+ } else if (isExpiredToken(ex)) {
+ throw newError(S3ErrorTable.EXPIRED_TOKEN, keyPath, ex);
+ } else if (isAccessDenied(ex)) {
+ throw newError(S3ErrorTable.ACCESS_DENIED, keyPath, ex);
+ } else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
+ throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else {
- throw ex;
+ throw newError(bucketName, keyPath, ex);
}
} catch (Exception ex) {
auditReadFailure(s3GAction, ex);
@@@ -749,14 -720,17 +726,17 @@@
final String uploadID = queryParams().get(QueryParams.UPLOAD_ID, "");
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.COMPLETE_MULTIPART_UPLOAD;
-- OzoneVolume volume = getVolume();
// Using LinkedHashMap to preserve ordering of parts list.
Map<Integer, String> partsMap = new LinkedHashMap<>();
List<CompleteMultipartUploadRequest.Part> partList =
multipartUploadRequest.getPartList();
+ S3ConditionalRequest.WriteConditions writeConditions =
+ S3ConditionalRequest.parseWriteConditions(getHeaders(), key);
+
OmMultipartUploadCompleteInfo omMultipartUploadCompleteInfo;
try {
++ OzoneVolume volume = getVolume();
OzoneBucket ozoneBucket = volume.getBucket(bucket);
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucket, ozoneBucket.getOwner());
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
index 009e22c42cb,2b6affd3c03..f3ff9f49464
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
@@@ -172,24 -131,20 +157,24 @@@ public class OS3Exception extends Runti
//When we get exception log it, and return exception as xml from actual
// exception data. So, falling back to construct from exception.
- String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
- "<Error>" +
- "<Code>%s</Code>" +
- "<Message>%s</Message>" +
- "<Resource>%s</Resource>" +
- "<RequestId>%s</RequestId>" +
- "</Error>";
- return String.format(formatString, this.getCode(),
- this.getErrorMessage(), this.getResource(),
- this.getRequestId());
+ final StringBuilder builder = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
+ .append("<Error>")
+ .append("<Code>").append(this.getCode()).append("</Code>")
+ .append("<Message>").append(this.getErrorMessage()).append("</Message>")
+ .append("<Resource>").append(this.getResource()).append("</Resource>")
+ .append("<RequestId>").append(this.getRequestId()).append("</RequestId>");
+ if (!Strings.isNullOrEmpty(this.getHostId())) {
+ builder.append("<HostId>").append(this.getHostId()).append("</HostId>");
+ }
+ if (!Strings.isNullOrEmpty(this.getToken0())) {
+ builder.append("<Token-0>").append(this.getToken0()).append("</Token-0>");
+ }
+ builder.append("</Error>");
+ return builder.toString();
}
- /** Create a copy with specific message. */
public OS3Exception withMessage(String message) {
- return new OS3Exception(code, message, httpCode);
+ setErrorMessage(message);
+ return this;
}
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
index 57c870a7eb0,00000000000..0604962cf76
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
@@@ -1,160 -1,0 +1,173 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents exceptions raised from Ozone STS service.
+ */
+public class OSTSException extends OS3Exception {
+ private static final Logger LOG = LoggerFactory.getLogger(OSTSException.class);
+ private static final ObjectMapper MAPPER;
+ private static final String AWS_FAULT_NS = "http://webservices.amazon.com/AWSFault/2005-15-09";
+ private static final String STS_NS = "https://sts.amazonaws.com/doc/2011-06-15/";
+ private static final String INVALID_ACTION = "InvalidAction";
+
+ static {
+ MAPPER = new XmlMapper();
+ MAPPER.registerModule(new JaxbAnnotationModule());
+ MAPPER.enable(SerializationFeature.INDENT_OUTPUT);
+ }
+
+ private String type = "Sender";
+
- public OSTSException(String codeVal, String messageVal, int httpCode) {
- super(codeVal, messageVal, httpCode);
++ public OSTSException(S3ErrorTable error) {
++ super(error, null, null);
+ }
+
- public OSTSException(String codeVal, String messageVal, int httpCode, String typeVal) {
- this(codeVal, messageVal, httpCode);
++ public OSTSException(S3ErrorTable error, Exception cause) {
++ super(error, cause, null);
++ }
++
++ @Override
++ public OSTSException withMessage(String message) {
++ super.withMessage(message);
++ return this;
++ }
++
++ /**
++ * STS fault party (e.g. {@code Sender} vs {@code Receiver}); defaults to {@code Sender}.
++ */
++ public OSTSException withType(String typeVal) {
+ this.type = typeVal;
++ return this;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String toXml() {
+ try {
+ final ErrorResponse response = new ErrorResponse(this);
+ final String val = MAPPER.writeValueAsString(response);
+ LOG.debug("toXml val is {}", val);
+ return val; // STS error responses don't have prolog <?xml version="1.0" encoding="UTF-8"?>
+ } catch (Exception ex) {
+ LOG.error("Exception occurred", ex);
+ // Fallback
+ final String namespace = INVALID_ACTION.equals(getCode()) ? AWS_FAULT_NS : STS_NS;
+
+ // STS error responses don't have prolog <?xml version="1.0" encoding="UTF-8"?>
+ final StringBuilder builder = new StringBuilder();
+ builder.append("<ErrorResponse xmlns=\"").append(namespace).append("\">\n")
+ .append(" <Error>\n")
+ .append(" <Type>").append(getType()).append("</Type>\n")
+ .append(" <Code>").append(getCode()).append("</Code>\n")
+ .append(" <Message>").append(getErrorMessage()).append("</Message>\n")
+ .append(" </Error>\n")
+ .append(" <RequestId>").append(getRequestId()).append("</RequestId>\n")
+ .append("</ErrorResponse>");
+ return builder.toString();
+ }
+ }
+
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlRootElement(name = "ErrorResponse")
+ private static class ErrorResponse {
+
+ @XmlAttribute
+ private String xmlns;
+
+ @XmlElement(name = "Error")
+ private ErrorDetails error;
+
+ @XmlElement(name = "RequestId")
+ private String requestId;
+
+ ErrorResponse() {
+ }
+
+ ErrorResponse(OSTSException ex) {
+ this.xmlns = INVALID_ACTION.equals(ex.getCode()) ? AWS_FAULT_NS : STS_NS;
+ this.error = new ErrorDetails(ex.getType(), ex.getCode(), ex.getErrorMessage());
+ this.requestId = ex.getRequestId();
+ }
+
+ public String getXmlns() {
+ return xmlns;
+ }
+
+ public ErrorDetails getError() {
+ return error;
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+ }
+
+ @XmlAccessorType(XmlAccessType.FIELD)
+ private static class ErrorDetails {
+ @XmlElement(name = "Type")
+ private String type;
+
+ @XmlElement(name = "Code")
+ private String code;
+
+ @XmlElement(name = "Message")
+ private String message;
+
+ ErrorDetails() {
+ }
+
+ ErrorDetails(String type, String code, String message) {
+ this.type = type;
+ this.code = code;
+ this.message = message;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+ }
+}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index f315012a976,022dc08949e..ee269b7542a
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@@ -32,122 -33,118 +33,122 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
/**
-- * This class represents errors from Ozone S3 service.
++ * This class represents errors from Ozone S3 service and STS-compatible errors
++ * raised by the S3 gateway STS endpoint.
* This class needs to be updated to add new errors when required.
*/
- public final class S3ErrorTable {
+ public enum S3ErrorTable {
- private static final Logger LOG = LoggerFactory.getLogger(
- S3ErrorTable.class);
+ INVALID_URI("InvalidURI",
+ "Couldn't parse the specified URI.", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_URI = new OS3Exception("InvalidURI",
- "Couldn't parse the specified URI.", HTTP_BAD_REQUEST);
+ NO_SUCH_VOLUME("NoSuchVolume",
+ "The specified volume does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception NO_SUCH_VOLUME = new OS3Exception(
- "NoSuchVolume", "The specified volume does not exist", HTTP_NOT_FOUND);
+ NO_SUCH_BUCKET("NoSuchBucket",
+ "The specified bucket does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception NO_SUCH_BUCKET = new OS3Exception(
- "NoSuchBucket", "The specified bucket does not exist", HTTP_NOT_FOUND);
+ AUTH_PROTOCOL_NOT_SUPPORTED("AuthProtocolNotSupported",
+ "Auth protocol used for this request is not supported.", HTTP_BAD_REQUEST),
- public static final OS3Exception AUTH_PROTOCOL_NOT_SUPPORTED =
- new OS3Exception("AuthProtocolNotSupported", "Auth protocol used for" +
- " this request is not supported.", HTTP_BAD_REQUEST);
-
- public static final OS3Exception S3_AUTHINFO_CREATION_ERROR =
- new OS3Exception("InvalidRequest", "Error creating s3 auth info. " +
+ S3_AUTHINFO_CREATION_ERROR("InvalidRequest", "Error creating s3 auth info. " +
"The request may not be signed using AWS V4 signing algorithm," +
- " or might be invalid", HTTP_FORBIDDEN);
+ " or might be invalid", HTTP_FORBIDDEN),
- public static final OS3Exception BUCKET_NOT_EMPTY = new OS3Exception(
+ BUCKET_NOT_EMPTY(
"BucketNotEmpty", "The bucket you tried to delete is not empty. " +
"If you are using --force option to delete all objects in the bucket, " +
"please ensure that the bucket layout is OBJECT_STORE or " +
"that the bucket is completely empty before delete.",
- HTTP_CONFLICT);
+ HTTP_CONFLICT),
- public static final OS3Exception MALFORMED_HEADER = new OS3Exception(
+ MALFORMED_HEADER(
"AuthorizationHeaderMalformed", "The authorization header you provided " +
- "is invalid.", HTTP_BAD_REQUEST);
+ "is invalid.", HTTP_BAD_REQUEST),
- public static final OS3Exception NO_SUCH_KEY = new OS3Exception(
- "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND);
+ NO_SUCH_KEY(
+ "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception INVALID_ARGUMENT = new OS3Exception(
- "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST);
+ INVALID_ARGUMENT(
+ "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_REQUEST = new OS3Exception(
- "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
+ INVALID_REQUEST(
+ "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_RANGE = new OS3Exception(
+ INVALID_RANGE(
"InvalidRange", "The requested range is not satisfiable",
- RANGE_NOT_SATISFIABLE);
+ RANGE_NOT_SATISFIABLE),
- public static final OS3Exception NO_SUCH_UPLOAD = new OS3Exception(
+ NO_SUCH_UPLOAD(
"NoSuchUpload", "The specified multipart upload does not exist. The " +
"upload ID might be invalid, or the multipart upload might have " +
- "been aborted or completed.", HTTP_NOT_FOUND);
+ "been aborted or completed.", HTTP_NOT_FOUND),
- public static final OS3Exception INVALID_BUCKET_NAME = new OS3Exception(
+ INVALID_BUCKET_NAME(
"InvalidBucketName", "The specified bucket is not valid.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_PART = new OS3Exception(
+ INVALID_PART(
"InvalidPart", "One or more of the specified parts could not be found." +
" The part might not have been uploaded, or the specified entity " +
- "tag might not have matched the part's entity tag.", HTTP_BAD_REQUEST);
+ "tag might not have matched the part's entity tag.", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_PART_ORDER = new OS3Exception(
+ INVALID_PART_ORDER(
"InvalidPartOrder", "The list of parts was not in ascending order. The " +
"parts list must be specified in order by part number.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception ENTITY_TOO_SMALL = new OS3Exception(
+ ENTITY_TOO_SMALL(
"EntityTooSmall", "Your proposed upload is smaller than the minimum " +
"allowed object size. Each part must be at least 5 MB in size, except " +
- "the last part.", HTTP_BAD_REQUEST);
+ "the last part.", HTTP_BAD_REQUEST),
- public static final OS3Exception INTERNAL_ERROR = new OS3Exception(
+ INTERNAL_ERROR(
"InternalError", "We encountered an internal error. Please try again.",
- HTTP_INTERNAL_ERROR);
+ HTTP_INTERNAL_ERROR),
- public static final OS3Exception ACCESS_DENIED = new OS3Exception(
+ ACCESS_DENIED(
"AccessDenied", "User doesn't have the right to access this " +
- "resource.", HTTP_FORBIDDEN);
+ "resource.", HTTP_FORBIDDEN),
- public static final OS3Exception EXPIRED_TOKEN = new OS3Exception(
- "ExpiredToken", "The provided token has expired.", HTTP_BAD_REQUEST);
++ EXPIRED_TOKEN(
++ "ExpiredToken", "The provided token has expired.", HTTP_BAD_REQUEST),
+
- public static final OS3Exception PRECOND_FAILED = new OS3Exception(
+ PRECOND_FAILED(
"PreconditionFailed", "At least one of the pre-conditions you " +
- "specified did not hold", HTTP_PRECON_FAILED);
+ "specified did not hold", HTTP_PRECON_FAILED),
+
+ CONDITIONAL_REQUEST_CONFLICT("ConditionalRequestConflict",
+ "A conflicting conditional operation occurred. Retry the request.",
+ HTTP_CONFLICT),
- public static final OS3Exception NOT_IMPLEMENTED = new OS3Exception(
+ NOT_IMPLEMENTED(
"NotImplemented", "This part of feature is not implemented yet.",
- HTTP_NOT_IMPLEMENTED);
+ HTTP_NOT_IMPLEMENTED),
- public static final OS3Exception NO_OVERWRITE = new OS3Exception(
- "Conflict", "Cannot overwrite file with directory", HTTP_CONFLICT);
+ NO_OVERWRITE(
+ "Conflict", "Cannot overwrite file with directory", HTTP_CONFLICT),
- public static final OS3Exception METADATA_TOO_LARGE = new OS3Exception(
+ METADATA_TOO_LARGE(
"MetadataTooLarge", "Illegal user defined metadata. Combined size " +
"exceeds the maximum allowed metadata size of " +
- S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB + "KB", HTTP_BAD_REQUEST);
+ S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB + "KB", HTTP_BAD_REQUEST),
- public static final OS3Exception BUCKET_ALREADY_EXISTS = new OS3Exception(
+ BUCKET_ALREADY_EXISTS(
"BucketAlreadyExists", "The requested bucket name is not available" +
- " as it already exists.", HTTP_CONFLICT);
+ " as it already exists.", HTTP_CONFLICT),
- public static final OS3Exception INVALID_TAG = new OS3Exception(
- "InvalidTag", "Your request contains tag input that is not valid.", HTTP_BAD_REQUEST);
+ INVALID_TAG(
+ "InvalidTag", "Your request contains tag input that is not valid.", HTTP_BAD_REQUEST),
- public static final OS3Exception NO_SUCH_TAG_SET = new OS3Exception(
- "NoSuchTagSet", "The specified tag does not exist.", HTTP_NOT_FOUND);
+ NO_SUCH_TAG_SET(
+ "NoSuchTagSet", "The specified tag does not exist.", HTTP_NOT_FOUND),
- public static final OS3Exception MALFORMED_XML = new OS3Exception(
+ MALFORMED_XML(
"MalformedXML", "The XML you provided was not well-formed or did not " +
- "validate against our published schema", HTTP_BAD_REQUEST);
+ "validate against our published schema", HTTP_BAD_REQUEST),
- public static final OS3Exception QUOTA_EXCEEDED = new OS3Exception(
+ QUOTA_EXCEEDED(
"QuotaExceeded", "The quota has been exceeded. " +
"Please review your disk space or namespace usage and adjust accordingly.",
HTTP_FORBIDDEN
@@@ -157,56 -154,128 +158,166 @@@
"InvalidStorageClass", "The storage class that you specified is not valid. " +
"Provide a supported storage class[STANDARD|REDUCED_REDUNDANCY|STANDARD_IA] or " +
"a valid custom EC storage config for if using STANDARD_IA.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception BUCKET_OWNER_MISMATCH = new OS3Exception(
+ BUCKET_OWNER_MISMATCH(
"Access Denied", "User doesn't have permission to access this resource due to a " +
- "bucket ownership mismatch.", HTTP_FORBIDDEN);
+ "bucket ownership mismatch.", HTTP_FORBIDDEN),
- public static final OS3Exception PAYLOAD_TOO_LARGE = new OS3Exception(
- "PayloadTooLarge", "Your request body size was too large.", HTTP_BAD_REQUEST
- );
++ PAYLOAD_TOO_LARGE(
++ "PayloadTooLarge", "Your request body size was too large.", HTTP_BAD_REQUEST),
+
- public static final OS3Exception X_AMZ_CONTENT_SHA256_MISMATCH = new OS3Exception(
+ X_AMZ_CONTENT_SHA256_MISMATCH(
"XAmzContentSHA256Mismatch", "The provided 'x-amz-content-sha256' header does " +
- "not match the computed hash.", HTTP_BAD_REQUEST);
+ "not match the computed hash.", HTTP_BAD_REQUEST),
- public static final OS3Exception BAD_DIGEST = new OS3Exception(
+ BAD_DIGEST(
"BadDigest", "The Content-MD5 or checksum value that you specified did not match what the server received.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
+
+ INVALID_DIGEST(
- "InvalidDigest", "The Content-MD5 you specified is not valid.", HTTP_BAD_REQUEST);
++ "InvalidDigest", "The Content-MD5 you specified is not valid.", HTTP_BAD_REQUEST),
++
++ /** STS: Code {@code ValidationError}; message is usually overridden via {@link OS3Exception#withMessage}. */
++ STS_VALIDATION_ERROR(
++ "ValidationError", "A validation error occurred.", HTTP_BAD_REQUEST),
++
++ /**
++ * STS: Code {@code InvalidAction} with HTTP 400 (unknown action or unsupported API version).
++ * For HTTP 501 unsupported-but-known operations use {@link #STS_INVALID_ACTION_NOT_IMPLEMENTED}.
++ */
++ STS_INVALID_ACTION(
++ "InvalidAction", "Could not find operation.", HTTP_BAD_REQUEST),
++
++ /** STS: Code {@code InvalidAction} with HTTP 501 (operation known but not implemented). */
++ STS_INVALID_ACTION_NOT_IMPLEMENTED(
++ "InvalidAction", "Operation is not supported yet.", HTTP_NOT_IMPLEMENTED),
++
++ /** STS: Code {@code InternalFailure}; distinct from {@link #INTERNAL_ERROR} ({@code InternalError}). */
++ STS_INTERNAL_FAILURE(
++ "InternalFailure", "An internal error has occurred.", HTTP_INTERNAL_ERROR),
+
- private static Function<Exception, OS3Exception> generateInternalError =
- e -> new OS3Exception("InternalError", e.getMessage(), HTTP_INTERNAL_ERROR);
++ /** STS: Code {@code InvalidClientTokenId}. */
++ STS_INVALID_CLIENT_TOKEN_ID(
++ "InvalidClientTokenId",
++ "The security token included in the request is invalid.", HTTP_FORBIDDEN),
+
- private S3ErrorTable() {
- //No one should construct this object.
++ /** STS: Code {@code UnsupportedOperation}; distinct from {@link #NOT_IMPLEMENTED} ({@code NotImplemented}). */
++ STS_UNSUPPORTED_OPERATION(
++ "UnsupportedOperation", "This operation is not supported.", HTTP_NOT_IMPLEMENTED),
++
++ /** STS: Code {@code MalformedPolicyDocument}; message is usually overridden via {@link OS3Exception#withMessage}. */
++ STS_MALFORMED_POLICY_DOCUMENT(
++ "MalformedPolicyDocument", "Policy document is malformed.", HTTP_BAD_REQUEST);
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3ErrorTable.class);
+
+ private final String code;
+ private final String message;
+ private final int httpCode;
+
+ S3ErrorTable(String code, String message, int httpCode) {
+ this.code = code;
+ this.message = message;
+ this.httpCode = httpCode;
}
- public static OS3Exception newError(OS3Exception e, String resource) {
- return newError(e, resource, null);
+ public String getCode() {
+ return code;
}
- /**
- * Create a new instance of Error.
- * @param e Error Template
- * @param resource Resource associated with this exception
- * @param ex the original exception, may be null
- * @return creates a new instance of error based on the template
- */
- public static OS3Exception newError(OS3Exception e, String resource,
- Exception ex) {
- OS3Exception err = new OS3Exception(e.getCode(), e.getErrorMessage(),
- e.getHttpCode());
- err.setResource(resource);
- if (e.getHttpCode() == HTTP_INTERNAL_ERROR) {
- LOG.error("Internal Error: {}", err.toXml(), ex);
- } else if (LOG.isDebugEnabled()) {
- LOG.debug(err.toXml(), ex);
+ public String getErrorMessage() {
+ return message;
+ }
+
+ public int getHttpCode() {
+ return httpCode;
+ }
+
+ /** Converts result code of {@code OMException} to {@code S3ErrorTable}, which
+ * should be thrown via {@link #newError(S3ErrorTable, String, Exception)}. */
+ public static S3ErrorTable translateResultCode(OMException ex) {
+ switch (ex.getResult()) {
+ case ACCESS_DENIED:
+ case INVALID_TOKEN:
+ case PERMISSION_DENIED:
++ case REVOKED_TOKEN:
+ return ACCESS_DENIED;
+ case ATOMIC_WRITE_CONFLICT:
+ return CONDITIONAL_REQUEST_CONFLICT;
+ case BUCKET_ALREADY_EXISTS:
+ return BUCKET_ALREADY_EXISTS;
+ case BUCKET_NOT_EMPTY:
+ return BUCKET_NOT_EMPTY;
+ case BUCKET_NOT_FOUND:
+ case VOLUME_NOT_FOUND:
+ return NO_SUCH_BUCKET;
+ case ENTITY_TOO_SMALL:
+ return ENTITY_TOO_SMALL;
+ case ETAG_MISMATCH:
+ case ETAG_NOT_AVAILABLE:
+ case KEY_ALREADY_EXISTS:
+ return PRECOND_FAILED;
++ case TOKEN_EXPIRED:
++ return EXPIRED_TOKEN;
+ case FILE_ALREADY_EXISTS:
+ return NO_OVERWRITE;
+ case INVALID_BUCKET_NAME:
+ return INVALID_BUCKET_NAME;
+ case INVALID_PART:
+ return INVALID_PART;
+ case INVALID_PART_ORDER:
+ return INVALID_PART_ORDER;
+ case INVALID_REQUEST:
+ return INVALID_REQUEST;
+ case KEY_NOT_FOUND:
+ return NO_SUCH_KEY;
+ case NOT_SUPPORTED_OPERATION:
+ return NOT_IMPLEMENTED;
+ case QUOTA_EXCEEDED:
+ return QUOTA_EXCEEDED;
+ default:
+ return INTERNAL_ERROR;
}
- return err;
}
- public static OS3Exception getInternalError(Exception e) {
- return generateInternalError.apply(e);
+ /** Same as {@link #newError(String, OMException)}, but uses {@code bucket} as the {@code resource}
+ * in case of {@link #NO_SUCH_BUCKET} error. */
+ public static OS3Exception newError(String bucket, String resource, OMException cause) {
+ S3ErrorTable errorCode = translateResultCode(cause);
+ return new OS3Exception(errorCode, cause, NO_SUCH_BUCKET == errorCode ? bucket : resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link OMException} and {@code resource}. */
+ public static OS3Exception newError(@Nullable String resource, OMException cause) {
+ return new OS3Exception(translateResultCode(cause), cause, resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable}. */
+ public static OS3Exception newError(S3ErrorTable errorCode) {
+ return new OS3Exception(errorCode, null, null);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable} and {@code resource}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, @Nullable String resource) {
+ return new OS3Exception(errorCode, null, resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable} and {@code cause}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, Exception cause) {
+ return new OS3Exception(errorCode, cause, null);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable}, {@code resource} and {@code cause}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, @Nullable String resource, Exception cause) {
+ return new OS3Exception(errorCode, cause, resource);
+ }
+
+ static void log(OS3Exception err) {
+ if (err.getHttpCode() == HTTP_INTERNAL_ERROR) {
+ LOG.error("Internal Error: {}", err.toXml(), err.getCause());
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug(err.toXml(), err.getCause());
+ }
}
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
index 343c341ee4f,02067537495..16acd86ec6f
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
@@@ -123,75 -96,6 +123,75 @@@ public class AWSSignatureProcessor impl
return signatureInfo;
}
+ private String extractSessionToken(LowerCaseKeyStringMap headers) {
+ // Header-based token
+ final String headerToken = headers.get(X_AMZ_SECURITY_TOKEN);
+ if (headerToken != null && !headerToken.isEmpty()) {
+ return headerToken;
+ }
+
+ // Query-based token - this would be used for presigned URLs
+ final MultivaluedMap<String, String> queryParams = context.getUriInfo().getQueryParameters();
+ if (queryParams == null) {
+ return null;
+ }
+ for (Map.Entry<String, List<String>> entry : queryParams.entrySet()) {
+ final String key = entry.getKey();
+ if (Strings.isNullOrEmpty(key)) {
+ continue;
+ }
+ if (key.compareToIgnoreCase(X_AMZ_SECURITY_TOKEN) == 0) {
+ final List<String> values = entry.getValue();
+ if (values != null && !values.isEmpty()) {
+ return values.get(0);
+ }
+ }
+ }
+ return null;
+ }
+
+ private String getPayloadHash(Map<String, String> headers, SignatureInfo signatureInfo)
+ throws OS3Exception, NoSuchAlgorithmException, IOException {
+ if (signatureInfo.getVersion() == Version.V2) {
- throw S3_AUTHINFO_CREATION_ERROR;
++ throw S3ErrorTable.newError(S3_AUTHINFO_CREATION_ERROR);
+ }
+ if (signatureInfo.getService().equals("s3")) {
+ if (!signatureInfo.isSignPayload()) {
+ // According to AWS Signature V4 documentation using Query Parameters
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+ return UNSIGNED_PAYLOAD;
+ }
+ String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256);
+ // According to AWS Signature V4 documentation using Authorization Header
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ // The x-amz-content-sha256 header is required
+ // for all AWS Signature Version 4 requests using Authorization header.
+ if (contentSignatureHeaderValue == null) {
+ LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
+ + " header for signed payload");
- throw S3_AUTHINFO_CREATION_ERROR;
++ throw S3ErrorTable.newError(S3_AUTHINFO_CREATION_ERROR);
+ }
+ // Simply return the header value of x-amz-content-sha256 as the payload hash
+ // These are the possible cases:
+ // 1. Actual payload checksum for single chunk upload
+ // 2. Unsigned payloads for multiple chunks upload
+ // - UNSIGNED-PAYLOAD
+ // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
+ // 3. Signed payloads for multiple chunks upload
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+ return contentSignatureHeaderValue;
+ }
+ // For STS payload hash is calculated over the body
+ InputStream in = context.getEntityStream();
+ byte[] body = readAllBytes(in);
+ String payloadHash = Hex.encode(MessageDigest.getInstance("SHA-256").digest(body));
+ context.setEntityStream(new ByteArrayInputStream(body));
+ return payloadHash;
+ }
+
private AuditMessage buildAuthFailureMessage(MalformedResourceException e) {
AuditMessage message = new AuditMessage.Builder()
.forOperation(AuthOperation.fromContext(context))
@@@ -203,22 -107,6 +203,21 @@@
return message;
}
+ private byte[] readAllBytes(InputStream in) throws OS3Exception, IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ byte[] chunk = new byte[8192];
+ int totalRead = 0;
+ int n;
+ while ((n = in.read(chunk)) != -1) {
+ if (totalRead + n > OZONE_S3G_STS_PAYLOAD_HASH_MAX_VALUE) {
- throw new OSTSException(
- PAYLOAD_TOO_LARGE.getCode(), PAYLOAD_TOO_LARGE.getErrorMessage(), PAYLOAD_TOO_LARGE.getHttpCode());
++ throw new OSTSException(PAYLOAD_TOO_LARGE);
+ }
+ buffer.write(chunk, 0, n);
+ totalRead += n;
+ }
+ return buffer.toByteArray();
+ }
+
@VisibleForTesting
public void setContext(ContainerRequestContext context) {
this.context = context;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index 9be7f7d6752,f308363133d..3eab4b1f33e
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@@ -19,6 -19,8 +19,7 @@@ package org.apache.hadoop.ozone.s3.sign
import static java.time.temporal.ChronoUnit.SECONDS;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
+ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import com.google.common.annotations.VisibleForTesting;
@@@ -173,15 -176,12 +175,15 @@@ public final class StringToSignProduce
StringBuilder canonicalHeaders = new StringBuilder();
for (String header : StringUtils.split(signedHeaders, ';')) {
- canonicalHeaders.append(header.toLowerCase());
- canonicalHeaders.append(':');
+ canonicalHeaders.append(header.toLowerCase())
+ .append(':');
if (headers.containsKey(header)) {
String headerValue = headers.get(header);
+ if (header.equals("content-type")) {
+ headerValue = headerValue.toLowerCase();
+ }
- canonicalHeaders.append(headerValue);
- canonicalHeaders.append(NEWLINE);
+ canonicalHeaders.append(headerValue)
+ .append(NEWLINE);
// Set for testing purpose only to skip date and host validation.
try {
@@@ -207,7 -210,38 +209,7 @@@
+ signedHeaders + NEWLINE
+ payloadHash;
}
-
- private static String getPayloadHash(Map<String, String> headers, boolean isUsingQueryParameter)
- throws OS3Exception {
- if (isUsingQueryParameter) {
- // According to AWS Signature V4 documentation using Query Parameters
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
- return UNSIGNED_PAYLOAD;
- }
- String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256);
- // According to AWS Signature V4 documentation using Authorization Header
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
- // The x-amz-content-sha256 header is required
- // for all AWS Signature Version 4 requests using Authorization header.
- if (contentSignatureHeaderValue == null) {
- LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
- + " header for signed payload");
- throw newError(S3_AUTHINFO_CREATION_ERROR);
- }
- // Simply return the header value of x-amz-content-sha256 as the payload hash
- // These are the possible cases:
- // 1. Actual payload checksum for single chunk upload
- // 2. Unsigned payloads for multiple chunks upload
- // - UNSIGNED-PAYLOAD
- // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
- // 3. Signed payloads for multiple chunks upload
- // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
- // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
- // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
- // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
- return contentSignatureHeaderValue;
- }
--
++
/**
* String join that also works with empty strings.
*
@@@ -324,11 -357,12 +325,11 @@@
*/
private static void validateCanonicalHeaders(
String canonicalHeaders,
- Map<String, String> headers,
- Boolean unsignedPaylod
- ) throws OS3Exception {
+ Map<String, String> headers)
+ throws OS3Exception {
if (!canonicalHeaders.contains(HOST + ":")) {
LOG.error("The SignedHeaders list must include HTTP Host header");
- throw S3_AUTHINFO_CREATION_ERROR;
+ throw newError(S3_AUTHINFO_CREATION_ERROR);
}
for (String header : headers.keySet().stream()
.filter(s -> s.startsWith("x-amz-"))
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
index 091f8851fa3,00000000000..1cc92230149
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
@@@ -1,354 -1,0 +1,344 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3sts;
+
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
- import static javax.ws.rs.core.Response.Status.FORBIDDEN;
- import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
- import static javax.ws.rs.core.Response.Status.NOT_IMPLEMENTED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INTERNAL_FAILURE;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_ACTION;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_ACTION_NOT_IMPLEMENTED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_CLIENT_TOKEN_ID;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_MALFORMED_POLICY_DOCUMENT;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_UNSUPPORTED_OPERATION;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_VALIDATION_ERROR;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.audit.S3GAction;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo;
+import org.apache.hadoop.ozone.om.helpers.AwsRoleArnValidator;
+import org.apache.hadoop.ozone.om.helpers.S3STSUtils;
+import org.apache.hadoop.ozone.s3.RequestIdentifier;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.OSTSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * AWS STS (Security Token Service) compatible endpoint for Ozone S3 Gateway.
+ * <p>
+ * This endpoint provides temporary security credentials compatible with
+ * AWS STS API, exposed on the port 9880 or 9881.
+ * <p>
+ * Currently supports only AssumeRole operation. Other STS operations will
+ * return appropriate error responses.
+ *
+ * @see <a href="https://docs.aws.amazon.com/STS/latest/APIReference/">AWS STS API Reference</a>
+ */
+@Path("/")
+@S3STSEnabled
+public class S3STSEndpoint extends S3STSEndpointBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3STSEndpoint.class);
+
+ // STS API constants
+ private static final String ASSUME_ROLE_ACTION = "AssumeRole";
+ private static final String GET_SESSION_TOKEN_ACTION = "GetSessionToken";
+ private static final String ASSUME_ROLE_WITH_SAML_ACTION = "AssumeRoleWithSAML";
+ private static final String ASSUME_ROLE_WITH_WEB_IDENTITY_ACTION = "AssumeRoleWithWebIdentity";
+ private static final String GET_CALLER_IDENTITY_ACTION = "GetCallerIdentity";
+ private static final String DECODE_AUTHORIZATION_MESSAGE_ACTION = "DecodeAuthorizationMessage";
+ private static final String GET_ACCESS_KEY_INFO_ACTION = "GetAccessKeyInfo";
+
+ private static final String EXPECTED_VERSION = "2011-06-15";
- private static final String VALIDATION_ERROR = "ValidationError";
- private static final String INVALID_ACTION = "InvalidAction";
- private static final String INTERNAL_FAILURE = "InternalFailure";
- private static final String ACCESS_DENIED = "AccessDenied";
- private static final String INVALID_CLIENT_TOKEN_ID = "InvalidClientTokenId";
- private static final String UNSUPPORTED_OPERATION = "UnsupportedOperation";
- private static final String MALFORMED_POLICY_DOCUMENT = "MalformedPolicyDocument";
+
+ // JAXBContext is relatively expensive to create and is threadsafe, so cache and reuse
+ private static final JAXBContext JAXB_CONTEXT;
+
+ static {
+ try {
+ JAXB_CONTEXT = JAXBContext.newInstance(S3AssumeRoleResponseXml.class);
+ } catch (JAXBException e) {
+ throw new RuntimeException("Failed to initialize JAXBContext: " + e, e);
+ }
+ }
+
+ @Inject
+ private RequestIdentifier requestIdentifier;
+
+ @VisibleForTesting
+ public void setRequestIdentifier(RequestIdentifier requestIdentifier) {
+ this.requestIdentifier = requestIdentifier;
+ }
+
+ /**
+ * STS endpoint that handles GET requests with query parameters.
+ * AWS STS supports both GET and POST requests.
+ *
+ * @param action The STS action to perform (AssumeRole, GetSessionToken, etc.)
+ * @param roleArn The ARN of the role to assume (for AssumeRole)
+ * @param roleSessionName Session name for the role (for AssumeRole)
+ * @param durationSeconds Duration of the token validity in seconds
+ * @param version AWS STS API version (should be "2011-06-15")
+ * @return Response containing STS response XML or error
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_XML)
+ public Response get(
+ @QueryParam("Action") String action,
+ @QueryParam("RoleArn") String roleArn,
+ @QueryParam("RoleSessionName") String roleSessionName,
+ @QueryParam("DurationSeconds") Integer durationSeconds,
+ @QueryParam("Version") String version,
+ @QueryParam("Policy") String awsIamSessionPolicy) throws OS3Exception {
+
+ return handleSTSRequest(action, roleArn, roleSessionName, durationSeconds, version, awsIamSessionPolicy);
+ }
+
+ /**
+ * STS endpoint that handles POST requests with form data.
+ * AWS STS typically uses POST requests with form-encoded parameters.
+ *
+ * @param action The STS action to perform
+ * @param roleArn The ARN of the role to assume
+ * @param roleSessionName Session name for the role
+ * @param durationSeconds Duration of the token validity
+ * @param version AWS STS API version
+ * @return Response containing STS response XML or error
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_XML)
+ public Response post(
+ @FormParam("Action") String action,
+ @FormParam("RoleArn") String roleArn,
+ @FormParam("RoleSessionName") String roleSessionName,
+ @FormParam("DurationSeconds") Integer durationSeconds,
+ @FormParam("Version") String version,
+ @FormParam("Policy") String awsIamSessionPolicy) throws OS3Exception {
+
+ return handleSTSRequest(action, roleArn, roleSessionName, durationSeconds, version, awsIamSessionPolicy);
+ }
+
+ private Response handleSTSRequest(String action, String roleArn, String roleSessionName,
+ Integer durationSeconds, String version, String awsIamSessionPolicy) throws OS3Exception {
+ final String requestId = requestIdentifier.getRequestId();
+ // NOTE: invalid, missing or unsupported actions are not added to the audit log
+ try {
+ if (action == null) {
+ // Amazon STS has a different structure for the XML error response when the action is missing
+ return Response.status(BAD_REQUEST)
+ .entity("<UnknownOperationException/>")
+ .type(MediaType.APPLICATION_XML)
+ .build();
+ }
+
+ switch (action) {
+ case ASSUME_ROLE_ACTION:
+ return handleAssumeRole(roleArn, roleSessionName, durationSeconds, awsIamSessionPolicy, version, requestId);
+ // These operations are not supported yet
+ case GET_SESSION_TOKEN_ACTION:
+ case ASSUME_ROLE_WITH_SAML_ACTION:
+ case ASSUME_ROLE_WITH_WEB_IDENTITY_ACTION:
+ case GET_CALLER_IDENTITY_ACTION:
+ case DECODE_AUTHORIZATION_MESSAGE_ACTION:
+ case GET_ACCESS_KEY_INFO_ACTION:
- throw new OSTSException(
- INVALID_ACTION, "Operation " + action + " is not supported yet.", NOT_IMPLEMENTED.getStatusCode());
++ throw new OSTSException(STS_INVALID_ACTION_NOT_IMPLEMENTED)
++ .withMessage("Operation " + action + " is not supported yet.");
+ default:
- throw new OSTSException(
- INVALID_ACTION, "Could not find operation " + action + " for version " +
- (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version),
- BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_INVALID_ACTION)
++ .withMessage("Could not find operation " + action + " for version " +
++ (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version));
+ }
+ } catch (OSTSException e) {
+ throw e;
+ } catch (Exception ex) {
+ LOG.error("Unexpected error during STS request", ex);
- throw new OSTSException(
- INTERNAL_FAILURE, "An internal error has occurred.", INTERNAL_SERVER_ERROR.getStatusCode(), "Receiver");
++ throw new OSTSException(STS_INTERNAL_FAILURE, ex).withType("Receiver");
+ }
+ }
+
+ private Response handleAssumeRole(String roleArn, String roleSessionName, Integer durationSeconds,
+ String awsIamSessionPolicy, String version, String requestId) throws OSTSException {
+ final String action = "AssumeRole";
+ final Map<String, String> auditParams = getAuditParameters();
+ S3STSUtils.addAssumeRoleAuditParams(
+ auditParams, roleArn, roleSessionName, awsIamSessionPolicy,
+ durationSeconds == null ? S3STSUtils.DEFAULT_DURATION_SECONDS : durationSeconds,
+ requestId);
+
+ // Validate parameters
+ if (version == null || !version.equals(EXPECTED_VERSION)) {
- final OSTSException exception = new OSTSException(
- INVALID_ACTION, "Could not find operation " + action + " for version " +
- (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version),
- BAD_REQUEST.getStatusCode());
++ final OSTSException exception = new OSTSException(STS_INVALID_ACTION)
++ .withMessage("Could not find operation " + action + " for version " +
++ (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version));
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, exception));
+ throw exception;
+ }
+
+ final Set<String> validationErrors = new HashSet<>();
+ int duration = durationSeconds == null ? S3STSUtils.DEFAULT_DURATION_SECONDS : durationSeconds;
+ try {
+ duration = S3STSUtils.validateDuration(durationSeconds);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ AwsRoleArnValidator.validateAndExtractRoleNameFromArn(roleArn);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ S3STSUtils.validateRoleSessionName(roleSessionName);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ if (LOG.isDebugEnabled() && StringUtils.isNotEmpty(awsIamSessionPolicy)) {
+ LOG.debug(
+ "AssumeRole requestId={} received Policy(len={}): {}", requestId, awsIamSessionPolicy.length(),
+ awsIamSessionPolicy);
+ }
+ S3STSUtils.validateSessionPolicy(awsIamSessionPolicy);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ final int numValidationErrors = validationErrors.size();
+ if (numValidationErrors > 0) {
+ //noinspection StringBufferReplaceableByString
- final StringBuilder builder = new StringBuilder();
- builder.append(numValidationErrors);
- builder.append(" validation ");
- builder.append(numValidationErrors > 1 ? "errors detected: " : "error detected: ");
- builder.append(String.join(";", validationErrors));
++ final StringBuilder builder = new StringBuilder()
++ .append(numValidationErrors)
++ .append(" validation ")
++ .append(numValidationErrors > 1 ? "errors detected: " : "error detected: ")
++ .append(String.join(";", validationErrors));
+ final String validationMessage = builder.toString();
- final OSTSException exception = new OSTSException(
- VALIDATION_ERROR, validationMessage, BAD_REQUEST.getStatusCode());
++ final OSTSException exception = new OSTSException(STS_VALIDATION_ERROR).withMessage(validationMessage);
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, exception));
+ throw exception;
+ }
+
+ final String assumedRoleUserArn = S3STSUtils.toAssumedRoleUserArn(roleArn, roleSessionName);
+ try {
+ final AssumeRoleResponseInfo responseInfo = getClient()
+ .getObjectStore()
+ .assumeRole(roleArn, roleSessionName, duration, awsIamSessionPolicy, requestId);
+ // Generate AssumeRole response
+ final String responseXml = generateAssumeRoleResponse(assumedRoleUserArn, responseInfo, requestId);
+
+ getAuditLogger().logWriteSuccess(buildAuditMessageForSuccess(S3GAction.ASSUME_ROLE, auditParams));
+
+ return Response.ok(responseXml)
+ .header("Content-Type", "text/xml")
+ .build();
+ } catch (IOException e) {
+ LOG.error("Error during AssumeRole processing", e);
+
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e));
+
+ if (e instanceof OMException) {
+ final OMException omException = (OMException) e;
+ if (omException.getResult() == OMException.ResultCodes.ACCESS_DENIED ||
+ omException.getResult() == OMException.ResultCodes.PERMISSION_DENIED ||
+ omException.getResult() == OMException.ResultCodes.TOKEN_EXPIRED) {
- throw new OSTSException(
- ACCESS_DENIED, "User is not authorized to perform: sts:AssumeRole on resource: " + roleArn,
- FORBIDDEN.getStatusCode());
++ throw new OSTSException(ACCESS_DENIED)
++ .withMessage("User is not authorized to perform: sts:AssumeRole on resource: " + roleArn);
+ }
+ if (omException.getResult() == OMException.ResultCodes.INVALID_TOKEN) {
- throw new OSTSException(
- INVALID_CLIENT_TOKEN_ID, "The security token included in the request is invalid.",
- FORBIDDEN.getStatusCode());
++ throw new OSTSException(STS_INVALID_CLIENT_TOKEN_ID);
+ }
+ if (omException.getResult() == OMException.ResultCodes.NOT_SUPPORTED_OPERATION ||
+ omException.getResult() == OMException.ResultCodes.FEATURE_NOT_ENABLED) {
- throw new OSTSException(UNSUPPORTED_OPERATION, omException.getMessage(), NOT_IMPLEMENTED.getStatusCode());
++ throw new OSTSException(STS_UNSUPPORTED_OPERATION).withMessage(omException.getMessage());
+ }
+ if (omException.getResult() == OMException.ResultCodes.INVALID_REQUEST) {
- throw new OSTSException(VALIDATION_ERROR, omException.getMessage(), BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_VALIDATION_ERROR).withMessage(omException.getMessage());
+ }
+ if (omException.getResult() == OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT) {
- throw new OSTSException(MALFORMED_POLICY_DOCUMENT, omException.getMessage(), BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_MALFORMED_POLICY_DOCUMENT).withMessage(omException.getMessage());
+ }
+ }
- throw new OSTSException(
- INTERNAL_FAILURE, "An internal error has occurred.", INTERNAL_SERVER_ERROR.getStatusCode(), "Receiver");
++ throw new OSTSException(STS_INTERNAL_FAILURE, e).withType("Receiver");
+ } catch (Exception e) {
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e));
+ throw e;
+ }
+ }
+
+ private String generateAssumeRoleResponse(String assumedRoleUserArn, AssumeRoleResponseInfo responseInfo,
+ String requestId) throws IOException {
+ final String accessKeyId = responseInfo.getAccessKeyId();
+ final String secretAccessKey = responseInfo.getSecretAccessKey();
+ final String sessionToken = responseInfo.getSessionToken();
+ final String assumedRoleId = responseInfo.getAssumedRoleId();
+
+ final String expiration = DateTimeFormatter.ISO_INSTANT.format(
+ Instant.ofEpochSecond(responseInfo.getExpirationEpochSeconds()).atOffset(ZoneOffset.UTC).toInstant());
+
+ try {
+ final S3AssumeRoleResponseXml response = new S3AssumeRoleResponseXml();
+ final S3AssumeRoleResponseXml.AssumeRoleResult result = new S3AssumeRoleResponseXml.AssumeRoleResult();
+ final S3AssumeRoleResponseXml.Credentials credentials = new S3AssumeRoleResponseXml.Credentials();
+ credentials.setAccessKeyId(accessKeyId);
+ credentials.setSecretAccessKey(secretAccessKey);
+ credentials.setSessionToken(sessionToken);
+ credentials.setExpiration(expiration);
+ result.setCredentials(credentials);
+ final S3AssumeRoleResponseXml.AssumedRoleUser user = new S3AssumeRoleResponseXml.AssumedRoleUser();
+ user.setAssumedRoleId(assumedRoleId);
+ user.setArn(assumedRoleUserArn);
+ result.setAssumedRoleUser(user);
+ response.setAssumeRoleResult(result);
+ final S3AssumeRoleResponseXml.ResponseMetadata meta = new S3AssumeRoleResponseXml.ResponseMetadata();
+ meta.setRequestId(requestId);
+ response.setResponseMetadata(meta);
+
+ final Marshaller marshaller = JAXB_CONTEXT.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ final StringWriter stringWriter = new StringWriter();
+ marshaller.marshal(response, stringWriter);
+ return stringWriter.toString();
+ } catch (JAXBException e) {
+ throw new IOException("Failed to marshal AssumeRole response", e);
+ }
+ }
+}
+
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
index fd272fe02fc,2644150bd34..97355f10e81
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
@@@ -48,37 -44,4 +45,36 @@@ public class TestOS3Exceptions
ex.getRequestId());
assertEquals(expected, val);
}
+
+ /**
+ * AWS S3 returns HTTP 400 Bad Request for ExpiredToken (not 403).
+ */
+ @Test
+ public void testExpiredTokenUsesBadRequestHttpStatus() {
+ assertEquals(HTTP_BAD_REQUEST, S3ErrorTable.EXPIRED_TOKEN.getHttpCode());
+ final OS3Exception fromTable = S3ErrorTable.newError(S3ErrorTable.EXPIRED_TOKEN, "resource");
+ assertEquals(HTTP_BAD_REQUEST, fromTable.getHttpCode());
+ }
+
+ @Test
+ public void testOS3ExceptionWithToken0() {
- OS3Exception ex = new OS3Exception("ExpiredToken", "The provided token has expired.", 400);
- ex = S3ErrorTable.newError(ex, "resource");
++ final OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.EXPIRED_TOKEN, "resource");
+ ex.setRequestId(OzoneUtils.getRequestID());
+ ex.setHostId(OzoneUtils.getRequestID());
+ ex.setToken0("token-value");
+
+ final String val = ex.toXml();
+ final String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>%n" +
+ "<Error>%n" +
+ " <Code>%s</Code>%n" +
+ " <Message>%s</Message>%n" +
+ " <Resource>%s</Resource>%n" +
+ " <RequestId>%s</RequestId>%n" +
+ " <HostId>%s</HostId>%n" +
+ " <Token-0>%s</Token-0>%n" +
+ "</Error>%n";
+ final String expected = String.format(formatString, ex.getCode(), ex.getErrorMessage(), ex.getResource(),
+ ex.getRequestId(), ex.getHostId(), ex.getToken0());
+ assertEquals(expected, val);
+ }
}
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
index cd77979ec20,00000000000..184e96f0406
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
@@@ -1,102 -1,0 +1,104 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.StringReader;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+/**
+ * This class tests OSTSException class. It is named TestOSTSExceptions instead of
+ * TestOSTSException to avoid findbugs rule about classes ending in *Exception must
+ * extend Exception or Throwable.
+ */
+public class TestOSTSExceptions {
+
+ @Test
+ public void testOSTSException() throws Exception {
- final OSTSException ex = new OSTSException("ValidationError", "1 validation error detected", 400);
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_VALIDATION_ERROR)
++ .withMessage("1 validation error detected");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ // Ensure the response uses the default namespace (no prefix like "ns2:")
+ assertEquals("ErrorResponse", root.getNodeName());
+ assertEquals("https://sts.amazonaws.com/doc/2011-06-15/", root.getNamespaceURI());
+
+ assertEquals("Sender", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("ValidationError", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("1 validation error detected", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ @Test
+ public void testOSTSExceptionInvalidAction() throws Exception {
- final OSTSException ex = new OSTSException("InvalidAction", "Could not find operation", 400);
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_INVALID_ACTION)
++ .withMessage("Could not find operation");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ assertEquals("http://webservices.amazon.com/AWSFault/2005-15-09", root.getNamespaceURI());
+
+ assertEquals("Sender", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("InvalidAction", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("Could not find operation", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ @Test
+ public void testOSTSExceptionWithCustomType() throws Exception {
- final OSTSException ex = new OSTSException("InternalFailure", "An internal error has occurred.", 500, "Receiver");
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_INTERNAL_FAILURE).withType("Receiver");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ assertEquals("https://sts.amazonaws.com/doc/2011-06-15/", root.getNamespaceURI());
+
+ assertEquals("Receiver", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("InternalFailure", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("An internal error has occurred.", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ private static Document parseXml(String xml) throws Exception {
+ assertNotNull(xml);
+ final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ documentBuilderFactory.setNamespaceAware(true);
+ final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+ return documentBuilder.parse(new InputSource(new StringReader(xml)));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org