You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2026/05/15 15:48:23 UTC
(ozone) branch HDDS-13323-sts updated (37a224b2176 -> 603a8abc375)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch HDDS-13323-sts
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 37a224b2176 HDDS-14899. [STS] Updates to ACLs in IamSessionPolicyResolver (#9977)
add 7d5c503d12b HDDS-14602. Bump pnpm to 10.28.2, Node.js to 20.18.2 (#9879)
add b0b2a25fab9 HDDS-14789. Compiler options not recognized by any processor (#9907)
add b5805af61e1 HDDS-14820. test-hadoop.sh fails locally due to bad substitution (#9906)
add ee81923da52 HDDS-14792. Bump commons-text to 1.15.0, commons-validator to 1.10.1 (#9890)
add bf01924cc7a HDDS-14828. Extract listStatus tests to OzoneFileSystemTestBase (#9917)
add 73072cf1dab HDDS-14832. Improve Datanode dispatcher unexpected container state message (#9918)
add 9d431fd8020 HDDS-13082. Add Jira account onboarding guidance (#9911)
add 3f47fa73287 HDDS-14833. Bump GitHub action versions (#9916)
add 317457d281c HDDS-14835. Bump maven-shade-plugin to 3.6.2 (#9923)
add 39a234752b1 HDDS-14823. Make PipelineID column sticky on Pipeline Tab (#9910)
add 3db9acb7a7e HDDS-14839. Bump commons-compress to 1.28.0 (#9924)
add 10d4da81b05 HDDS-14836. Bump awssdk to 2.42.13 (#9920)
add 3304e940914 HDDS-14838. Bump maven-core to 3.9.14 (#9921)
add 102b51a319c HDDS-14837. Bump opentelemetry to 1.60.1 (#9922)
add 020452e196b HDDS-14701. Consolidate DiskBalancerVolumeChoosingPolicy and ContainerChoosingPolicy (#9858)
add 3f36857b5bb HDDS-14740. Implement ScmListCodec without reflection (#9919)
add 886dd588e92 HDDS-14850. Implement StatefulService without reflection.. (#9940)
add cc801988596 HDDS-14848. OzoneManagerSyncMetrics incrNumDeltaRequestsFailed increments the wrong value (#9936)
add 990af8e1469 HDDS-14805. Escape log4j variables in docker-compose env_files (#9930)
add fb0c7d001d4 HDDS-14788. Avoid using guava Ints/Longs to convert byte arrays (#9927)
add ccf67f6d2b2 HDDS-14849. LowerCaseKeyStringMap.remove() should transform key toLowerCase (#9937)
add 243be86725c HDDS-14768. Fix lock leak during snapshot cache cleanup and handle eviction race appropriately. (#9869)
add b49226d1b57 HDDS-14860. Synchronize ThrottledAsyncChecker#schedule (#9948)
add 99e971cbf28 HDDS-14852. Create one MessageUnmarshaller per type (#9944)
add 137ea80bc34 HDDS-14800. Guard RocksDB iterator against closed DB during volume failure (#9904)
add c5f1dd35b53 HDDS-14857. Use XMLUtils.newSecure...Factory (#9951)
add fa3fcc8897a HDDS-14765. Improve Trace Hierarchy for Ozone Shell Put Key Command (#9908)
add 0afee11a137 HDDS-14778. Append dot to relocation patterns in ozone-filesystem-shaded (#9893)
add e2ad852dbd2 HDDS-14684. Allow deletion of empty quasi-closed containers (#9856)
add ead5648f914 HDDS-14880. Bump junit to 5.14.3 (#9962)
add 98ce7e761de HDDS-14875. Remove the type parameter from ScmCodec.deserialize(...) (#9956)
add 57b6084fcf6 HDDS-14879. Bump sqlite-jdbc to 3.51.3.0 (#9961)
add 6cdc9fcc57b HDDS-14878. Bump maven-clean-plugin to 3.5.0 (#9958)
add 40e893ff3eb HDDS-14853. Reduce duplicate logs in ECReconstructionCoordinator#reconstructECBlockGroup (#9941)
add 8813d8cfe3e HDDS-14827. Sync insightsFilePlot chart labels with sorted filesize data (#9912)
add 2ebb1509771 HDDS-14831. Datanode should gracefully handle CONTAINER_ALREADY_EXISTS (#9925)
add f1a5fe31369 HDDS-14882. Exclude transitive dependency commons-daemon via hadoop-hdfs (#9965)
add 9c97ef29ab7 HDDS-14846. Implement ScmCodecFactory without reflection (#9952)
add 412f9f45076 HDDS-14844. Update reconOmTasks memory counter using init after reinit (#9933)
add b93bb437d2e HDDS-14883. ContainerBalancerStatus shows 0GB due to incorrect field mapping (#9968)
add 448357940bc HDDS-14858. OM request fails with ClassNotFoundException: java.lang.constant.Constable (#9945)
add de1b3523d03 HDDS-13891. SCM-based health monitoring and batch processing in Recon (#9258)
add 987e3bcbb4b HDDS-14106. Add -XX:NewRatio=3 to default GC options for CMS (#9967)
add fe80c6f0eb9 HDDS-14885. Replace usage of Hadoop ...TestUtils (#9966)
add 5e5243eca2a HDDS-14714. Support keeping a configurable number of extra copies of quasi-closed containers. (#9829)
add 06a7d1295d1 HDDS-14752. ReadBlock: Close stream immediately when position >= blockLength (#9946)
add 7ee78a270f4 HDDS-14730. Update Recon container sync to use container IDs (#9842)
add 0a384a01d72 HDDS-14881. Exclude dependencies for test-jar (#9970)
add 533ef7ab417 HDDS-14887. BASH_SOURCE0: unbound variable (#9975)
add 44bf21645a5 HDDS-14834. Fix race condition between DeadNodeHandler and HealthyReadOnlyNodeHandler on NetworkTopology (#9926)
add d499272132c HDDS-14814. Unify fragmented traces for Freon randomkeys command (#9957)
add 3cc7c5d5dda HDDS-14862. Log volume failure as error (#9950)
add 7ca17344833 HDDS-14904. Fix outdated links to Ozone downloads (#9982)
add f3cd59a89c0 HDDS-14183. Attempted to decrement available space to a negative value (#9655)
add 6f04298a549 HDDS-14884. Improve Trace Hierarchy for Freon ockg Command (#9980)
add 7b6eeb4a379 HDDS-14370. RandomAccessFileChannel to implement Closeable (#9905)
add 99f4132744c HDDS-14682. Unify OzoneManagerProtocolPB failover proxy provider (#9929)
add c6da3cf1abd HDDS-13963. Atomic Create-If-Not-Exists (#9332)
add 1a3909286ae HDDS-14909. Bump jersey2 to 2.48 (#9989)
add 397c3d61b19 HDDS-14911. Bump maven-javadoc-plugin to 3.12.0 (#9991)
add 6dca71bfd59 HDDS-14910. Bump jackson to 2.21.2 (#9992)
add 7ee4a7077d1 HDDS-14912. Bump maven-source-plugin to 3.4.0 (#9993)
add 9f5565833a6 HDDS-12992. Clients should not use gRPC port for Streaming (#9988)
add 64db6553518 HDDS-14892. Define a LocalHandler interface (#9995)
add a01ad898922 HDDS-14916. Add a code generator for implementing invokeLocal(..). (#9996)
add c7ca682dd65 HDDS-14840. User Experience improvements on Buckets tab (#9928)
add ecd89f2feaf HDDS-14901. Enable Dependabot for github-actions (#9984)
add 501c0ff19b9 HDDS-14918. Enable PMD rules StringToString, UseArraysAsList and UseIndexOfChar (#9999)
add a7ba744889f HDDS-14611. Rename committed bytes to finalized key bytes and align committed space terminology (#9871)
add 28ec57ae916 HDDS-14928. Bump jline to 3.30.9 (#10002)
add 1ca91920d9f HDDS-14929. Bump awssdk to 2.42.18 (#10005)
add 1b83e169815 HDDS-14866. Enhance DiskBalancer Report to show individual volume's density (#9969)
add 3c3ec7de276 HDDS-14931. Bump commons-codec to 1.21.0 (#10003)
add d8e9f3dbef9 HDDS-14930. Bump maven-resources-plugin to 3.5.0 (#10004)
add dfede3a7b2a HDDS-14727. Improve volume/bucket selection in File Size Distribution (#9865)
add 4a3c626a73b HDDS-14917. Remove unused version definitions (#9998)
add 644a018e63a HDDS-14948. Pin axios version to 1.13.6 (#10016)
add 561cbeb3b24 HDDS-14941. Speed up testRetryMessageShownWhenScmUnavailable (#10019)
add e9cceba8dfc HDDS-13919. S3 Conditional Writes (PutObject) (#9815)
add 5e009a36360 HDDS-14890. Improve ozone_add_default_gc_opts (#10021)
add b111e0072f2 HDDS-14824. Fix/Freeze Hostname column in Recon v2 Datanode Tab (#10026)
add 8fbe16b4efa HDDS-14795. Allow non-S3-compliant bucket name length if strict S3 is disabled (#10014)
add 6a3286904ab HDDS-13919. S3 Conditional Writes (PutObject) [2/2] - Reuse Atomic Rewrite at Commit Path (#10023)
add 033e0bb0824 HDDS-14934. Enable PMD rule ConsecutiveAppendsShouldReuse (#10008)
add 5bc0bf8788e HDDS-13661. Fix flaky TestKeyDeletingService#testPurgeKeysRequestBatching (#10025)
add 65b938ef924 HDDS-14922. Define ScmInvoker#invokeRatisServer to proxy DeletedBlockLogStateManager without reflection (#10013)
add dd27370c84e HDDS-14971. Bump log4j2 to 2.25.4 (#10041)
add 2224f763105 HDDS-14793. Intermittent NPE in XceiverClientGrpc#connectToDatanode due to race condition (#9997)
add f72c54c2bbf HDDS-14972. Bump awssdk to 2.42.23 (#10039)
add 8b4e3ce7118 HDDS-14970. Bump jackson2 to 2.21.2 (#10040)
add c0d50fe17d2 HDDS-14961. Introduce configuration for tracing (#10029)
add 93bf1d9d16c HDDS-14975. Move awssdk version definition to ozone-integration-test-s3 (#10042)
add b02ebe2ac96 HDDS-10916. Reduce Recon OM snapshot task initial delay in acceptance tests (#10048)
add 994a8abaafc HDDS-14976. xml_to_md.py does not handle generated XML files (#10044)
add ce303c9277d HDDS-14919. Add a base class for all the SCM handler interfaces (#10050)
add ece57e53c89 HDDS-14978. Bump Zookeeper to 3.8.6 (#10046)
add 5c53d2e5915 HDDS-14041. Add metrics to track Snapshot RocksDB space and SST File stats. (#9406)
add 0c3751acfa4 HDDS-14963. Retry attempt to pull docker images (#10047)
add 346b65e9371 HDDS-14870. Allow balancing of over replicated and quasi closed containers (#9964)
add 9e775e6c8f6 HDDS-14973. Update ScmInvokerCodeGenerator to generate getProxy() (#10049)
add 33a5320e4ec HDDS-7373. Add Robot tests for `ozone --validate classpath` (#10051)
add 9ff7eaa6c19 HDDS-13108. Refactor StorageVolume to use SlidingWindow (#8843)
add 49d6c0b6cbd HDDS-14103. Create an option to suppress/unsuppress containers from report (#9719)
add d2d374b6749 HDDS-8703. Integration test for SnapshotDeletingService during OM failover (#10024)
add ab71c6af45f HDDS-14756. Implement delegation and auditing for BucketOperationHandler (#10057)
add b6ba3691421 HDDS-14447. Support multiple clients in OmMetadataGenerator (#9696)
add 45ffbf389cd HDDS-14660. Implement new table to store part information for multipart uploads (#9886)
add 4d8c38d47b4 HDDS-14843. Support cluster-wide blacklist on OM (#10037)
add 914b99cd7e7 HDDS-14968. Concurrent S3 Conditional PUT Commit Conflict Detection (#10043)
add 3c1e005a44e HDDS-13804. Make server tracing config dynamically reconfigurable (#10056)
add 4501218e086 HDDS-15003. Inline EndpointBase#createS3Bucket/deleteS3Bucket (#10060)
add 675410e0934 HDDS-13920. Conditional Reads (GetObject, HeadObject) (#10031)
add e8fac639e21 HDDS-14938. Implement Iceberg RewriteTablePath action (#10053)
add 02877735177 HDDS-15021. Bump jnr-posix to 3.1.22 (#10070)
add f99b4f63b0b HDDS-14998. Use line.separator in option descriptions (#10063)
add 203f7ee1bae HDDS-15020. Bump awssdk to 2.42.28 (#10069)
add 830305c8603 HDDS-15019. Bump commons-configuration2 to 2.14.0 (#10068)
add a650a90e622 HDDS-15023. S3ErrorTable should be an enum (#10072)
add 584bb6a3cb4 HDDS-14955. Improve BigInteger instantiation (#10035)
add 9f3d7cb1bbe HDDS-14960. OM Web UI dashboard for Ozone Snapshot (#10027)
add e407268752d HDDS-14996. Add metrics to track Snapshot backup SST file stats. (#10065)
add 3daabc8d99c HDDS-15013. gRPC channel holding objects on completed request in ReadBlock (#10071)
add ea503067ac6 HDDS-15032. Return InvalidDigest instead of BadDigest for malformed Content-MD5 header (#10078)
add 29c78e7ac11 HDDS-12643. Fix ContainerTable pagination in Recon UI v2 (#9986)
add 1bde5bea1d4 HDDS-15041. Fix some typos (#10080)
add 57ec56b5169 HDDS-14775. Check leader status before serving bootstrap request. (#9873)
add 191c01c346b HDDS-3128. Add ozone debug commands for Kerberos (#9868)
add 3650eb90303 HDDS-13145. Tweak javadoc plugin configuration (#9704)
add 95028e4ab0d HDDS-15024. Track pending containers in SCM to prevent Datanode over-allocation (#10073)
add 678118f99fc HDDS-15050. Bump Bouncy Castle to 1.84 (#10083)
add 9b44eb93e01 HDDS-14974. Change SCMStateMachine to use ScmInvoker (#10079)
add e744b818552 HDDS-14990. Show failed volumes in `ozone admin datanode list` output and SCM metrics (#10058)
add 1139d937a12 HDDS-15054. Bump guava to 33.6.0-jre (#10089)
add c183f27d42a HDDS-15055. Bump opentelemetry to 1.61.0 (#10088)
add e700ef726f9 HDDS-15057. Bump actions/cache to 5.0.4 (#10087)
add be2a9154457 HDDS-15056. Bump awssdk to 2.42.33 (#10090)
add f1122aa1c55 HDDS-14939. Implement version file rewrite logic for path migration across metadata history (#10067)
add a469057d2b1 HDDS-15042. Run basic checks with Java 21 (#10093)
add 0039ce6ea39 HDDS-15062. Bump actions/cache to 5.0.5 (#10091)
add 954141886db HDDS-15060. [Recon] Fix totalOzoneUsedSpace mismatch in storageDistribution (#10092)
add 8eb6a7be378 HDDS-14829. Split snapshot diff job into separate rpc calls for submitting job and getting the report. (#9985)
add 31c18dff19c HDDS-15000. Improve S3 audit log stack traces (#10097)
add 719bab13bba HDDS-15078. Reduce duplication in BucketAclHandler (#10106)
add 5d6c4f00f48 HDDS-14999. Use line.separator in cli-admin (#10107)
add bee32d5b52a HDDS-15040. Fix inverted ExecutorService#awaitTermination (#10113)
add b3b4f320bd9 HDDS-14987. OM Web UI dashboard for Ozone Snapshot (list snapshot) (#10055)
add 7eaf46fba46 HDDS-14871. DataNode: tolerate per-volume health-check latch timeouts before marking volumes failed. (#9954)
add d5ec23e417a HDDS-10819. Respect ssl.server.include.cipher.list and ssl.enabled.protocols in HttpServer2 (#10111)
add 543a744b79b HDDS-15102. Avoid ArchiveOutputStream.createArchiveEntry due to libnss issue (#10117)
add d64cbf66923 HDDS-15117. Bump awssdk to 2.42.36 (#10136)
add e0e821f6310 HDDS-13117. Add Conditional Writes, Reads, Copies, Deletes and CompleteMultipartUpload Design (#10010)
add 653775965fe HDDS-15108. Remove duplicate keys from ozone-default.xml (#10121)
add 6d0be38772d HDDS-15052. Revert HDDS-14040. Ozone client hang for data write in failure scenario (#10140)
add 86f3df176f6 HDDS-15103. OM termination due to race condition between om start and install snapshot (#10118)
add bf4515f16ca HDDS-14926. Allow QUASI_CLOSED containers in DiskBalancer with improved debug logs for containers (#10022)
add 9fb9bbe3d88 HDDS-15115. CrcUtil/CrcComposer should not throw IOException for non-IO. (#10139)
add bc54891616a HDDS-14920. Check actions with zizmor (#10102)
add d85b05aad64 HDDS-15081. Add Internal Scaffold (#10146)
add 04d1044e552 HDDS-14920. (addendum) Check actions with zizmor (#10150)
add be1ed89a302 HDDS-15104. Refactor code related to container space management. (#10124)
add 0f82ff6b9d8 HDDS-15107. Bump actions/upload-artifact to 7.0.1 (#10151)
add c408f6eb118 HDDS-15094. Make protocol and cipher configurable for gRPC TLS (#10114)
add 83abee4d71c HDDS-15105. Revise snapshot defrag design doc and add implementation details (#10119)
add 659fc3107b9 HDDS-15136. Missing OZONE_WEBSITE_BUILD in update-ozone-site-config-doc (#10154)
add b3f4a645dee HDDS-15112. Handle negative space usage values gracefully in CachingSpaceUsageSource (#10130)
add f7615733a44 HDDS-15118. Bump maven-core to 3.9.15 (#10137)
add 72e69f09697 HDDS-15124. Fix broken /docs/current/ links (#10153)
add 0c19a9b931d HDDS-15076. Fix Incorrect pending deletion size for EC blocks in DN (#10105)
add be850c1130b HDDS-15080. DirectoryDeletingService is using single thread (#10125)
add 5797d63c4c9 HDDS-15119. Bump sqlite-jdbc to 3.53.0.0 (#10135)
add b5083649171 HDDS-10489. Use CRC tables to speed up galoisFieldMultiply in CrcUtil. (#10155)
add fdd20f0d9e7 HDDS-15064. [STS] Artifacts for Ranger to Consider S3 Action when Authorizing (#10108)
add 39aeb20f072 HDDS-15100. Add an OM config to toggle Ozone snapshot rename feature (#10156)
add 99b53f44436 HDDS-13133. Display Ratis state machine event timeline in OM web UI (#10034)
add e215b48bfdd HDDS-10767. Reducing DatanodeDetails in the ContainerLocationCache (#10158)
add 9e89ee799e1 HDDS-15026. Introduce DevContainer to improve contributor experience (#10075)
add 9ccc1d63e45 HDDS-15157. Add DeepWiki badge to README (#10179)
add f07fb4e028c HDDS-15162. Bump awssdk to 2.42.41 (#10173)
add 0d191fe18b7 HDDS-15163. Bump commons-io to 2.22.0 (#10172)
add 329bc816682 HDDS-15161. Bump commons-codec to 1.22.0 (#10174)
add 1bf9a3bfdc6 HDDS-15044. Exercise ozone debug kerberos commands (#10085).
add 33697280560 HDDS-15067. Conditional CompleteMultipartUpload (#10164)
add 8507816f18b HDDS-13919. S3 Conditional Writes (PutObject) [3/3] - Remove expectedETag from OmKeyInfo (#10182)
add acce8f4364a HDDS-15053. Update ScmInvokerCodeGenerator for generating the latest invokeLocal (#10180)
add f64ef284666 HDDS-15033. Link SCM allocate-block calls to client trace. (#10081)
add 95e1b3c87d2 HDDS-15106. Add Delete block calls to trace hierarchy. (#10141)
add ae1599ba79d HDDS-15159. Bump common-custom-user-data-maven-extension to 2.2.0 (#10176)
add a4fcd65c31f HDDS-15160. Bump javassist to 3.31.0-GA (#10175)
add e4d87731730 HDDS-14942. Implement manifest selection logic for rewrite based on snapshot delta (#10145)
add 0be20f17879 HDDS-14894. Fix Latent S3 API Issue having No Acl Check for ListMultipartUploads (#9971)
add c3ffd0562e8 HDDS-15178. Allow creating EncodingTypeObject for empty String (#10195)
add 5b90ae700d1 HDDS-14830. Handle interrupt gracefully in XceiverClientGrpc.sendCommandWithRetry (#10178)
add 7f4069b0ee1 HDDS-11838. Top-level interactive shell to allow access to admin/debug/sh commands (#10134)
add aa76a5dc936 HDDS-15184. Atomic Create-If-Absent Should Use 0 for Generation Match (#10201)
add 83e77c09a5d HDDS-14966. OM UI shows wrong Leader Readiness values (#10036)
add f71c648e710 HDDS-14898. Fix Latent S3 API Issue having No Acl Check for ListParts (#9976)
add 0f811d443ce HDDS-15127. Add a switch to toggle the RocksDb open test for volume health (#10149)
add bc89991f35f HDDS-15004. Stabilize TestReconContainerEndpoint#testContainerEndpointForOBSBucket (#10116)
add debc8abf484 HDDS-15176. Ozone SCM fails to start when gRPC cipher policy list includes unsupported cipher (#10192)
add cf76290db43 HDDS-15109. Extract rename test cases to OzoneFileSystemTests (#10120)
add bb93dd2ebc8 HDDS-15152. SSL protocol config is not applied to Jetty when set to default value (#10165)
add 837813c14c3 HDDS-14923. Enable PMD rules UselessStringValueOf, UseStringBufferForStringAppends (#10007)
add ec6dd34d679 HDDS-15111. Remove unused ChecksumByteBuffer implementations from PureJava CRC helpers (#10127)
add d33f9943a5e HDDS-15177. Wait for healthy KDC before starting Ozone in xcompat tests (#10194)
add 8adb40535e0 HDDS-14913. Implement Scalable CSV Export for Unhealthy Containers in Recon UI. (#10162).
add 95867e884fd HDDS-14578. Ozone admin command gives inconsistent error messages on expired keytab. (#10160)
add 652bb192e58 HDDS-15110. Fix negative container size handling in replication (#10126)
add 32a88e76aa6 HDDS-15205. Cut WritableRPCEngine (#10213)
add ea66b70462c HDDS-15201. Record lastDefragTime when snapshot defrag is successfully run (#10210)
add f0323d7f4fc HDDS-13921. Conditional Copy (CopyObject) (#10207)
add 0ac31fbac01 HDDS-15214. Bump jackson to 2.21.3 (#10221)
add 233caf240dc HDDS-15214. Bump jackson to 2.21.3 (#10220)
add 35d394b25ae HDDS-14977. Mark TestDeadNodeHandler#onMessage as flaky
add 7ca74efa76c HDDS-15221. Mark TestReconWithOzoneManagerHA as flaky
add 77fd78ba0e8 HDDS-15209. Mark TestRocksDBCheckpointDiffer as flaky
add ae99c6a394f HDDS-14962. Mark TestContainerStateMachine#testApplyTransactionFailure as flaky
add a03f0fd45b4 HDDS-13401. Mark TestContainerCommandReconciliation as flaky
add e5c27ac6f07 HDDS-15222. Mark TestOzoneManagerHASnapshot#testSnapshotNameConsistency as flaky
add 44fffb51c62 HDDS-14017. Mark TestAddRemoveOzoneManager#testDecommission as flaky
add 6e7a0091e0f HDDS-15223. Mark TestReconAndAdminContainerCLI#testMissingContainer as flaky
add 7a024da2259 HDDS-15218. Bump zstd-jni to 1.5.7-8 (#10219)
add 95723c14bda HDDS-15217. Bump junit to 5.14.4 (#10223)
add 489629965b4 HDDS-15203. Clarify ozone.om.namespace.s3.strict in ozone-default.xml (#10225)
add 7b935fe4a7e HDDS-15168. Support configurable SASL mechanism (#10212)
add d1dbbe46574 HDDS-15149. Limit Connections Created by DataNode GRPC Server (#10184)
add 7dd0072d143 HDDS-15216. Remove duplicate jackson2 version property (#10229)
add e136cb00bfb HDDS-14943. Implement rewrite logic for Iceberg's manifest-list files for path migration (#10190)
add c3872cdb02d HDDS-11199. Fix eslint configs and add missing packages for Recon web (#10094)
add 6cd1da736d6 HDDS-15219. Bump awssdk to 2.44.0 (#10222)
add e5da18a959c HDDS-15082. Add User facing Config Contract for `ozone local` (#10147)
add 99328c24239 HDDS-15181. Robot test for snapshot defrag (#10200)
new 603a8abc375 Merge remote-tracking branch 'upstream/master' into HDDS-13323-sts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.devcontainer/devcontainer.json | 29 +
.github/dependabot.yml | 10 +
.github/pull_request_template.md | 3 +
.github/workflows/build-ratis.yml | 32 +-
.github/workflows/check.yml | 32 +-
.github/workflows/ci-with-ratis.yml | 9 +-
.github/workflows/ci.yml | 67 +-
.github/workflows/close-stale-prs.yaml | 4 +-
.github/workflows/generate-config-doc.yml | 11 +-
.github/workflows/intermittent-test-check.yml | 64 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 14 +-
.github/workflows/post-commit.yml | 7 +-
.github/workflows/pull-request.yml | 6 +-
.github/workflows/repeat-acceptance.yml | 22 +-
.github/workflows/schedule-label-pr.yml | 5 +-
.github/workflows/scheduled-cache-update.yml | 3 +-
.github/workflows/update-ozone-site-config-doc.yml | 45 +-
.github/workflows/{pull-request.yml => zizmor.yml} | 34 +-
.mvn/extensions.xml | 2 +-
CONTRIBUTING.md | 5 +-
README.md | 11 +-
SECURITY.md | 2 +-
dev-support/ci/xml_to_md.py | 31 +-
dev-support/pmd/pmd-ruleset.xml | 7 +
hadoop-hdds/cli-common/pom.xml | 16 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 33 +-
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 20 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 154 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 8 +
.../hdds/scm/storage/StreamBlockInputStream.java | 48 +-
.../io/ECBlockReconstructedStripeInputStream.java | 8 +-
.../scm/storage/TestStreamBlockInputStream.java | 193 +-
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 9 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 -
hadoop-hdds/common/pom.xml | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 22 +
.../org/apache/hadoop/hdds/client/BlockID.java | 4 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 10 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 37 +
.../balancer/ContainerBalancerConfiguration.java | 34 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hadoop/hdds/security/SecurityConfig.java | 36 +
.../apache/hadoop/hdds/tracing/LoopSampler.java | 38 +-
.../apache/hadoop/hdds/tracing/SkipTracing.java} | 9 +-
.../apache/hadoop/hdds/tracing/SpanSampler.java | 90 +
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 31 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 128 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 160 +-
.../apache/hadoop/hdds/utils/SlidingWindow.java | 11 +-
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 21 +-
.../hdds/utils/io/RandomAccessFileChannel.java | 37 +-
.../org/apache/hadoop/ipc_/AlignmentContext.java | 2 +-
.../src/main/java/org/apache/hadoop/ipc_/RPC.java | 18 +-
.../main/java/org/apache/hadoop/ipc_/Server.java | 51 +-
.../org/apache/hadoop/ipc_/WritableRpcEngine.java | 630 --
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 3 +
.../org/apache/hadoop/ozone/common/Checksum.java | 4 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 78 -
.../ozone/common/PureJavaCrc32ByteBuffer.java | 23 +-
.../ozone/common/PureJavaCrc32CByteBuffer.java | 23 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../security_/CustomizedCallbackHandler.java | 121 +
.../hadoop/security_/SaslMechanismFactory.java | 90 +
.../org/apache/hadoop/security_/SaslRpcClient.java | 16 +-
.../org/apache/hadoop/security_/SaslRpcServer.java | 63 +-
.../common/src/main/resources/ozone-default.xml | 230 +-
.../security/TestSecurityConfigTlsSettings.java | 90 +
.../hadoop/hdds/tracing/TestSpanSampling.java | 144 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 34 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 50 +-
.../hdds/utils/io/TestRandomAccessFileChannel.java | 153 +
.../ozone/common/TestChecksumByteBuffer.java | 4 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +-
hadoop-hdds/config/pom.xml | 10 +
.../hadoop/hdds/conf/ConfigFileAppender.java | 8 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 2 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 12 +-
.../ozone/container/common/impl/ContainerData.java | 48 +-
.../container/common/impl/ContainerDataYaml.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 237 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 6 +-
.../states/endpoint/VersionEndpointTask.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 13 +-
.../server/ratis/ContainerStateMachine.java | 22 +-
.../ozone/container/common/volume/HddsVolume.java | 44 +-
.../container/common/volume/MutableVolumeSet.java | 9 +-
.../container/common/volume/StorageVolume.java | 121 +-
.../common/volume/StorageVolumeChecker.java | 114 +-
.../common/volume/ThrottledAsyncChecker.java | 2 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ContainerChoosingPolicyFactory.java | 55 +
.../diskbalancer/DiskBalancerConfiguration.java | 138 +-
.../container/diskbalancer/DiskBalancerInfo.java | 55 +-
.../diskbalancer/DiskBalancerProtocolServer.java | 5 +-
.../diskbalancer/DiskBalancerService.java | 160 +-
.../container/diskbalancer/DiskBalancerYaml.java | 20 +-
...ChoosingPolicy.java => ContainerCandidate.java} | 42 +-
.../policy/ContainerChoosingPolicy.java | 36 +-
.../policy/DefaultContainerChoosingPolicy.java | 208 +-
.../policy/DefaultVolumeChoosingPolicy.java | 156 -
.../ECReconstructionCoordinator.java | 10 -
.../keyvalue/impl/FilePerBlockStrategy.java | 27 +-
.../ozone/container/metadata/AbstractRDBStore.java | 14 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 4 +-
.../container/replication/ReplicationServer.java | 5 +
.../ScmHAFinalizeUpgradeActionDatanode.java | 1 +
.../commands/ReplicateContainerCommand.java | 10 +-
.../container/common/impl/TestHddsDispatcher.java | 43 +
.../statemachine/TestDatanodeConfiguration.java | 32 +
.../server/ratis/TestContainerStateMachine.java | 45 +-
.../common/volume/TestStorageVolumeChecker.java | 185 +
.../volume/TestStorageVolumeHealthChecks.java | 149 +-
.../TestDefaultContainerChoosingPolicy.java | 780 +-
.../TestDefaultVolumeChoosingPolicy.java | 651 --
.../TestDiskBalancerProtocolServer.java | 65 +-
.../diskbalancer/TestDiskBalancerService.java | 81 +-
.../diskbalancer/TestDiskBalancerTask.java | 22 +-
.../diskbalancer/TestDiskBalancerYaml.java | 79 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 51 +
.../TestBackgroundContainerDataScanner.java | 114 +
hadoop-hdds/docs/content/design/diskbalancer.md | 16 +-
.../docs/content/design/s3-conditional-requests.md | 668 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 43 +-
.../docs/content/feature/DiskBalancer.zh.md | 25 +-
.../feature/Snapshot-Configuration-Properties.md | 1 +
hadoop-hdds/docs/content/feature/Snapshot.md | 9 +-
.../content/feature/SnapshotDefragmentation.md | 212 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 23 +-
hadoop-hdds/framework/pom.xml | 13 +-
.../hdds/conf/TracingReconfigurationCallback.java | 58 +
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 12 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 29 +
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHandler.java} | 16 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 16 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 37 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 57 +-
.../symmetric/DefaultSecretKeySignerClient.java | 2 +-
.../hdds/security/symmetric/SecretKeyState.java | 9 +-
.../certificate/authority/CertificateStore.java | 8 +-
.../client/DefaultCertificateClient.java | 6 +-
.../certificate/utils/SelfSignedCertificate.java | 2 +-
.../apache/hadoop/hdds/server/OzoneBlacklist.java | 219 +
.../server/OzoneProtocolMessageDispatcher.java | 3 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 33 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 38 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 15 +-
.../apache/hadoop/hdds/utils/HttpServletUtils.java | 2 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 14 +-
.../hdds/utils/db/RDBStoreAbstractIterator.java | 19 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 4 +
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 23 +-
.../org/apache/hadoop/ozone/common/BlockGroup.java | 9 +-
.../apache/hadoop/ozone/common/DeletedBlock.java | 17 +-
.../src/main/resources/webapps/static/ozone.js | 2 +
.../resources/webapps/static/templates/menu.html | 2 +
.../hdds/fs/TestCachingSpaceUsageSource.java | 35 +
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 2 +-
...ctionWithReload.java => TestGrpcTlsConfig.java} | 214 +-
.../hdds/server/http/TestHttpServer2SSL.java | 361 +
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 18 +-
.../hdds/utils/db/TestFixedLengthStringCodec.java | 7 +-
.../utils/db/TestRDBStoreIteratorWithDBClose.java | 308 +
.../src/main/proto/ScmAdminProtocol.proto | 28 +
.../src/main/proto/RpcHeader.proto | 2 +-
.../interface-client/src/main/proto/hdds.proto | 16 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../utils/db/managed/ManagedRocksIterator.java | 31 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 5 +
.../scm/block/DeletedBlockLogStateManager.java | 33 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 50 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 39 +-
.../hdds/scm/container/ContainerManagerImpl.java | 40 +-
.../hdds/scm/container/ContainerStateManager.java | 41 +-
.../scm/container/ContainerStateManagerImpl.java | 44 +-
.../scm/container/balancer/ContainerBalancer.java | 20 +-
.../ContainerBalancerSelectionCriteria.java | 107 +-
.../container/balancer/ContainerBalancerTask.java | 2 +
.../hdds/scm/container/balancer/MoveManager.java | 109 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 -
.../container/placement/metrics/SCMMetrics.java | 36 +-
...nQueue.java => MonitoringReplicationQueue.java} | 4 +-
.../QuasiClosedStuckOverReplicationHandler.java | 4 +-
.../replication/QuasiClosedStuckReplicaCount.java | 77 +-
.../QuasiClosedStuckUnderReplicationHandler.java | 13 +-
.../replication/RatisUnderReplicationHandler.java | 9 +
.../container/replication/ReplicationManager.java | 74 +-
.../replication/health/EmptyContainerHandler.java | 74 +-
.../health/QuasiClosedStuckReplicationCheck.java | 20 +-
.../scm/container/states/ContainerStateMap.java | 14 +
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 +
.../apache/hadoop/hdds/scm/ha/ReflectionUtil.java | 67 -
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 10 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 35 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 36 +-
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 10 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 7 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 57 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 48 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 8 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 4 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 328 +
.../DeletedBlockLogStateManagerInvoker.java | 148 +
.../ha/invoker/PipelineStateManagerInvoker.java | 282 +
.../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 82 +
.../scm/ha/invoker/ScmInvokerCodeGenerator.java | 698 ++
.../hadoop/hdds/scm/ha/invoker}/package-info.java | 5 +-
.../hadoop/hdds/scm/ha/io/ScmBigIntegerCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmBooleanCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmByteStringCodec.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/io/ScmCodec.java | 3 +-
.../hadoop/hdds/scm/ha/io/ScmCodecFactory.java | 122 +-
.../apache/hadoop/hdds/scm/ha/io/ScmEnumCodec.java | 8 +-
.../hadoop/hdds/scm/ha/io/ScmIntegerCodec.java | 10 +-
.../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java | 84 +-
.../apache/hadoop/hdds/scm/ha/io/ScmLongCodec.java | 10 +-
.../hdds/scm/ha/io/ScmManagedSecretKeyCodec.java | 10 +-
.../scm/ha/io/ScmNonShadedByteStringCodec.java | 2 +-
.../ha/io/ScmNonShadedGeneratedMessageCodec.java | 10 +-
.../hadoop/hdds/scm/ha/io/ScmStringCodec.java | 2 +-
.../hdds/scm/ha/io/ScmX509CertificateCodec.java | 8 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 10 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 39 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 14 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 13 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hdds/scm/node/PendingContainerTracker.java | 225 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 30 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 24 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 -
.../hdds/scm/pipeline/PipelineStateManager.java | 14 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 5 +-
...inerLocationProtocolServerSideTranslatorPB.java | 53 +-
.../hdds/scm/security/RootCARotationHandler.java | 9 +-
.../scm/security/RootCARotationHandlerImpl.java | 4 +-
.../hdds/scm/security/RootCARotationManager.java | 17 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 109 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 20 +-
.../hdds/scm/server/StorageContainerManager.java | 27 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/server/StorageContainerManagerStarter.java | 2 -
.../server/upgrade/FinalizationStateManager.java | 9 +-
.../upgrade/FinalizationStateManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 5 +-
.../main/resources/webapps/scm/ratis-events.html} | 22 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../src/main/resources/webapps/scm/scm.js | 26 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 1 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 37 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../scm/container/TestContainerManagerImpl.java | 19 +-
.../scm/container/TestContainerReportHandler.java | 101 +-
.../scm/container/TestContainerStateManager.java | 44 +-
.../TestContainerBalancerSelectionCriteria.java | 188 +-
.../scm/container/balancer/TestMoveManager.java | 175 +
.../algorithms/TestContainerPlacementFactory.java | 2 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 24 +
...TestQuasiClosedStuckOverReplicationHandler.java | 96 +-
.../TestQuasiClosedStuckReplicaCount.java | 302 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 61 +-
.../health/TestEmptyContainerHandler.java | 36 +
.../TestQuasiClosedStuckReplicationCheck.java | 40 +-
.../container/states/TestContainerStateMap.java | 76 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 19 -
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 4 +-
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 +
.../hdds/scm/ha/io/TestScmBigIntegerCodec.java | 2 +-
.../hadoop/hdds/scm/ha/io/TestScmListCodec.java} | 30 +-
.../scm/ha/io/TestScmX509CertificateCodec.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 185 +
.../hdds/scm/node/TestPendingContainerTracker.java | 381 +
.../hdds/scm/node/states/TestNodeStateMap.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 11 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 56 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +-
.../scm/security/TestRootCARotationManager.java | 2 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 3 +-
.../scm/server/TestSCMClientProtocolServer.java | 31 +-
.../replicationManagerTests/quasi_closed.json | 57 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 6 +-
.../apache/ozone/test/TimedOutTestsListener.java | 4 +-
hadoop-ozone/cli-admin/pom.xml | 13 +
.../scm/cli/ContainerBalancerStatusSubcommand.java | 21 +-
.../hdds/scm/cli/ContainerOperationClient.java | 21 +
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 5 +-
.../scm/cli/container/ContainerIDParameters.java | 14 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 12 +-
.../hdds/scm/cli/container/ListSubcommand.java | 9 +-
.../scm/cli/container/ReconcileSubcommand.java | 26 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 65 +-
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 54 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 6 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 4 +-
.../cli/datanode/DiskBalancerReportSubcommand.java | 145 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 11 +
.../cli/datanode/DiskBalancerStatusSubcommand.java | 23 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 15 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 62 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 4 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 14 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 11 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 4 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 13 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 8 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 2 +-
.../ozone/admin/om/ListOpenFilesSubCommand.java | 61 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 18 +-
.../ozone/admin/scm/RotateKeySubCommand.java | 2 +-
.../TestContainerReportSuppressOptions.java | 310 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 5 +-
.../scm/cli/container/TestReconcileSubcommand.java | 50 +-
.../datanode/TestContainerBalancerSubCommand.java | 2 +-
.../cli/datanode/TestDiskBalancerSubCommands.java | 88 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 62 +-
hadoop-ozone/cli-debug/pom.xml | 20 +-
.../ozone/debug/audit/parser/AuditParser.java | 4 +-
.../debug/audit/parser/common/DatabaseHelper.java | 4 +-
.../audit/parser/handler/LoadCommandHandler.java | 6 +-
.../audit/parser/handler/QueryCommandHandler.java | 6 +-
.../parser/handler/TemplateCommandHandler.java | 16 +-
.../ozone/debug/kerberos/AuthorizationProbe.java | 106 +
.../hadoop/ozone/debug/kerberos/ConfigProbe.java | 116 +
.../ozone/debug/kerberos/DiagnoseSubcommand.java | 113 +
.../ozone/debug/kerberos/DiagnosticProbe.java | 18 +-
.../ozone/debug/kerberos/EnvironmentProbe.java} | 28 +-
.../hadoop/ozone/debug/kerberos/HostProbe.java | 72 +
.../hadoop/ozone/debug/kerberos/HttpAuthProbe.java | 52 +
.../ozone/debug/kerberos/JvmKerberosProbe.java | 98 +
.../ozone/debug/kerberos/KerberosConfigProbe.java | 61 +
.../KerberosSubcommand.java} | 17 +-
.../ozone/debug/kerberos/KerberosTicketProbe.java | 95 +
.../hadoop/ozone/debug/kerberos/KeytabProbe.java | 93 +
.../hadoop/ozone/debug/kerberos/KinitProbe.java | 61 +
.../debug/kerberos/PrincipalMappingProbe.java | 113 +
.../hadoop/ozone/debug/kerberos/ProbeResult.java | 9 +-
.../ozone/debug/kerberos/SecurityConfigProbe.java | 72 +
.../kerberos/TranslatePrincipalSubcommand.java | 78 +
.../hadoop/ozone/debug/kerberos}/package-info.java | 5 +-
.../debug/kerberos/TestOzoneDebugKerberos.java | 159 +
.../hadoop/ozone/debug/kerberos}/package-info.java | 4 +-
hadoop-ozone/cli-repair/pom.xml | 16 +-
.../schemaupgrade/ContainerUpgradeResult.java | 36 +-
.../schemaupgrade/VolumeUpgradeResult.java | 28 +-
hadoop-ozone/cli-shell/pom.xml | 16 +-
.../hadoop/ozone/shell/ListPaginationOptions.java | 2 +-
.../hadoop/ozone/shell/OzoneInteractiveShell.java | 84 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 26 +-
.../apache/hadoop/ozone/shell/acl/AclOption.java | 4 +-
.../hadoop/ozone/shell/bucket/BucketUri.java | 2 +-
.../ozone/shell/bucket/DeleteBucketHandler.java | 4 +-
.../hadoop/ozone/shell/common/VolumeBucketUri.java | 2 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 2 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 10 +-
.../hadoop/ozone/shell/prefix/PrefixUri.java | 2 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 43 +-
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 2 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 2 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +-
.../hadoop/ozone/shell/volume/VolumeUri.java | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 49 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 111 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 16 +
.../ozone/client/io/KeyDataStreamOutput.java | 6 +-
.../ozone/client/protocol/ClientProtocol.java | 138 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 249 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 9 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 36 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 4 +-
.../hadoop/ozone/client/checksum/CrcComposer.java | 79 +-
.../hadoop/ozone/client/checksum/CrcUtil.java | 155 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +-
.../hadoop/ozone/om/exceptions/OMException.java | 9 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 55 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 32 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 166 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 349 +
.../ozone/om/helpers/OmMultipartPartKey.java | 233 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 4 +-
.../ozone/om/helpers/OzoneFileStatusLight.java | 4 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 21 +
.../apache/hadoop/ozone/om/helpers/S3STSUtils.java | 16 +-
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 15 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 50 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 9 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 61 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 103 +-
.../ozone/security/acl/AssumeRoleRequest.java | 28 +-
.../hadoop/ozone/security/acl/RequestContext.java | 20 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 59 +-
.../ozone/snapshot/SubmitSnapshotDiffResponse.java | 65 +
.../org/apache/hadoop/ozone/util/RadixTree.java | 4 +-
.../ozone/client/checksum/TestCrcComposer.java | 222 +
.../hadoop/ozone/client/checksum/TestCrcUtil.java | 236 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 41 +
.../ozone/om/helpers/TestOmMultipartPartInfo.java | 143 +
.../ozone/om/helpers/TestOmMultipartPartKey.java | 204 +
.../om/helpers/TestOmSnapshotDiffJobCodec.java | 2 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 24 +
.../ozone/security/acl/TestAssumeRoleRequest.java | 73 +-
.../acl/iam/TestIamSessionPolicyResolver.java | 26 +-
.../hadoop/ozone/security/acl/package-info.java | 2 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 78 +
.../snapshot/TestSubmitSnapshotDiffResponse.java | 97 +
hadoop-ozone/dist/README.md | 6 +-
hadoop-ozone/dist/pom.xml | 15 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +
.../dist/src/main/compose/common/hadoop-test.sh | 5 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 2 +-
.../src/main/compose/ozone-om-ha/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 4 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +-
.../compose/ozonesecure-ha/test-debug-tools.sh | 2 +
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 26 +
.../main/compose/upgrade/compose/ha/docker-config | 1 +
.../compose/upgrade/compose/non-ha/docker-config | 1 +
.../dist/src/main/compose/xcompat/new-cluster.yaml | 3 +
.../dist/src/main/compose/xcompat/old-cluster.yaml | 3 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 7 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 8 +-
.../dist/src/main/smoketest/cli/classpath.robot | 47 +-
.../src/main/smoketest/compatibility/setup.robot | 9 +-
.../src/main/smoketest/compatibility/write.robot | 65 +
.../smoketest/debug/ozone-debug-kerberos.robot | 59 +
.../omha/data-validation-after-om-bootstrap.robot | 15 +-
.../src/main/smoketest/s3/conditionalput.robot | 89 +
.../main/smoketest/scmha/container-create.robot | 2 +-
.../main/smoketest/snapshot/snapshot-defrag.robot | 172 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 12 +
.../dist/src/shell/ozone/ozone-functions.sh | 16 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 5 -
.../network-tests/src/test/compose/docker-config | 4 +-
hadoop-ozone/freon/pom.xml | 20 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 18 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 143 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 2 +-
hadoop-ozone/httpfsgateway/pom.xml | 32 -
.../src/main/resources/httpfs-default.xml | 8 -
.../iceberg}/dev-support/findbugsExcludeFile.xml | 0
.../erasurecode => hadoop-ozone/iceberg}/pom.xml | 83 +-
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 463 +
.../ozone/iceberg/RewriteTablePathOzoneUtils.java | 126 +
.../apache/hadoop/ozone/iceberg}/package-info.java | 4 +-
.../iceberg/TestRewriteTablePathOzoneAction.java | 361 +
hadoop-ozone/integration-test-recon/pom.xml | 15 +
...> AbstractTestStorageDistributionEndpoint.java} | 282 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 1 +
.../ozone/recon/TestReconContainerEndpoint.java | 80 +-
.../ozone/recon/TestReconOmMetaManagerUtils.java | 33 +
.../apache/hadoop/ozone/recon/TestReconTasks.java | 802 +-
.../ozone/recon/TestReconTasksMultiNode.java | 178 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +
.../recon/TestStorageDistributionEndpointEC.java | 93 +
.../TestStorageDistributionEndpointRatis.java | 100 +
hadoop-ozone/integration-test-s3/pom.xml | 17 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 464 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 698 +-
.../src/test/resources/log4j.properties | 0
hadoop-ozone/integration-test/pom.xml | 25 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 20 +-
.../fs/contract/AbstractContractConcatTest.java | 8 +-
.../AbstractContractContentSummaryTest.java | 4 +-
.../AbstractContractCopyFromLocalTest.java | 10 +-
.../AbstractContractGetFileStatusTest.java | 10 +-
.../AbstractContractLeaseRecoveryTest.java | 14 +-
.../AbstractContractMultipartUploaderTest.java | 55 +-
.../fs/contract/AbstractContractOpenTest.java | 74 +-
.../AbstractContractRootDirectoryTest.java | 18 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 306 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 299 +-
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 323 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 144 +
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 51 +
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 1 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 8 +-
.../hdds/scm/TestStorageContainerManager.java | 11 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 27 +
.../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +-
.../TestReplicationManagerIntegration.java | 154 +
.../TestScmDataDistributionFinalization.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 8 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 333 +-
.../rpc/TestContainerStateMachineFailures.java | 65 +-
.../replication/TestContainerReplication.java | 53 +
.../TestContainerCommandReconciliation.java | 3 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 6 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 1 +
...stOzoneManagerHAFollowerReadWithAllRunning.java | 5 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 82 +
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 2 +
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../ozone/reconfig/TestOmReconfiguration.java | 74 +
.../ozone/reconfig/TestScmReconfiguration.java | 2 +
.../scm/node/TestContainerChoosingPolicy.java | 300 -
.../node/TestDiskBalancerPolicyPerformance.java | 381 +
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 -
.../ozone/shell/TestDeletedBlocksTxnShell.java | 3 +-
.../tools/contract/AbstractContractDistCpTest.java | 14 +-
.../src/test/resources/log4j.properties | 6 -
.../src/main/proto/OmClientProtocol.proto | 51 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +
hadoop-ozone/ozone-manager/pom.xml | 15 -
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 5 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 24 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 21 +
.../om/OMDBCheckpointServletInodeBasedXfer.java | 14 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 61 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 12 +-
.../hadoop/ozone/om/OmSnapshotLocalData.java | 2 +-
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 15 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 61 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 266 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 -
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 57 +-
...ONServlet.java => SnapshotListJSONServlet.java} | 82 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 28 +-
.../hadoop/ozone/om/ha/OMPeriodicMetrics.java | 131 +
.../ozone/om/ratis/OzoneManagerStateMachine.java | 49 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 34 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 27 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 96 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 7 +
.../request/snapshot/OMSnapshotRenameRequest.java | 10 +
.../ozone/om/service/DirectoryDeletingService.java | 30 +-
.../ozone/om/service/KeyDeletingService.java | 8 +-
.../om/snapshot/OMSnapshotDirectoryMetrics.java | 328 +
.../om/snapshot/OmSnapshotLocalDataManager.java | 25 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 37 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 214 +-
.../om/snapshot/SnapshotDiffManagerMXBean.java} | 16 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 62 +-
.../hadoop/ozone/security/STSTokenIdentifier.java | 12 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 47 +
.../main/resources/webapps/ozoneManager/index.html | 4 +-
.../webapps/ozoneManager/om-snapshots.html | 176 +
.../resources/webapps/ozoneManager/ozoneManager.js | 155 +
.../webapps/ozoneManager/ratis-events.html | 22 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 80 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 2 +
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 50 +-
.../TestOzoneManagerListMultipartUploadsAcls.java | 196 +
.../ozone/om/TestOzoneManagerListPartsAcls.java | 196 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 55 +-
.../ozone/om/TestSnapshotListJSONServlet.java | 57 +
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 25 +
.../TestOmRatisSnapshotProvider.java | 8 +-
.../request/TestOMClientRequestWithUserInfo.java | 2 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +-
.../request/bucket/TestOMBucketCreateRequest.java | 35 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 77 +-
.../om/request/key/TestOMKeyCreateRequest.java | 235 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 +-
.../s3/security/TestS3GetSecretRequest.java | 2 +-
.../s3/tenant/TestOMTenantCreateRequest.java | 2 +-
.../s3/tenant/TestOMTenantDeleteRequest.java | 2 +-
.../snapshot/TestOMSnapshotRenameRequest.java | 20 +
.../validation/TestOMValidatorProcessor.java | 16 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +-
.../om/service/TestDirectoryDeletingService.java | 94 +-
.../ozone/om/service/TestKeyDeletingService.java | 37 +-
.../om/service/TestSnapshotDiffCleanupService.java | 2 +-
.../snapshot/TestOMSnapshotDirectoryMetrics.java | 125 +
.../snapshot/TestOmSnapshotLocalDataManager.java | 10 +
.../ozone/om/snapshot/TestSnapshotCache.java | 143 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 317 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 156 +
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 87 +
.../ozone/security/acl/TestOzoneBlacklist.java | 235 +
.../ozone/security/acl/TestRequestContext.java | 6 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 61 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 67 +-
hadoop-ozone/pom.xml | 9 +
.../ozone/recon/codegen/TableNamingStrategy.java | 12 +-
.../recon/schema/ContainerSchemaDefinition.java | 20 +-
hadoop-ozone/recon/pom.xml | 14 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 71 +
.../ozone/recon/api/ClusterStateEndpoint.java | 7 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 268 +-
.../hadoop/ozone/recon/api/ExportJobManager.java | 404 +
.../recon/api/StorageDistributionEndpoint.java | 69 +-
.../ozone/recon/api/handlers/BucketHandler.java | 14 +-
.../hadoop/ozone/recon/api/types/ExportJob.java | 222 +
.../ozone/recon/api/types/GlobalStorageReport.java | 20 +-
.../ozone/recon/api/types/UsedSpaceBreakDown.java | 12 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 793 +-
.../fsck/NoOpsContainerReplicaPendingOps.java | 112 +
.../ozone/recon/fsck/ReconReplicationManager.java | 661 ++
.../recon/fsck/ReconReplicationManagerReport.java | 66 +
.../recon/metrics/ContainerHealthMetrics.java | 86 -
.../recon/metrics/ContainerHealthTaskMetrics.java | 75 +
.../recon/metrics/OzoneManagerSyncMetrics.java | 2 +-
.../persistence/ContainerHealthSchemaManager.java | 596 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 128 +-
.../recon/scm/ReconStorageContainerSyncHelper.java | 113 +
.../recon/spi/StorageContainerServiceProvider.java | 8 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 9 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 16 +-
.../impl/StorageContainerServiceProviderImpl.java | 9 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 3 +-
.../ozone/recon/upgrade/ReconLayoutFeature.java | 4 +-
...ntainersStateContainerIdIndexUpgradeAction.java | 86 +
.../webapps/recon/ozone-recon-web/.eslintrc.json | 180 +-
.../webapps/recon/ozone-recon-web/api/db.json | 2736 ++++-
.../recon/ozone-recon-web/api/pagination.js | 86 +
.../webapps/recon/ozone-recon-web/api/routes.json | 5 +
.../webapps/recon/ozone-recon-web/package.json | 10 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 10840 +++++++++++--------
.../mocks/capacityMocks/capacityResponseMocks.ts | 4 +-
.../src/v2/components/plots/insightsFilePlot.tsx | 18 +-
.../src/v2/components/select/multiSelect.tsx | 134 +-
.../v2/components/select/multiSelectMenuList.tsx | 160 +
.../src/v2/components/select/singleSelect.tsx | 1 +
.../src/v2/components/tables/bucketsTable.tsx | 2 +
.../src/v2/components/tables/containersTable.tsx | 71 +-
.../src/v2/components/tables/datanodesTable.tsx | 1 +
.../src/v2/components/tables/pipelinesTable.tsx | 2 +-
.../src/v2/constants/capacity.constants.tsx | 4 +-
.../src/v2/constants/select.constants.tsx | 7 +-
.../src/v2/pages/buckets/buckets.tsx | 9 +-
.../src/v2/pages/capacity/capacity.tsx | 4 +-
.../src/v2/pages/containers/containers.tsx | 816 +-
.../src/v2/pages/insights/insights.tsx | 1 +
.../ozone-recon-web/src/v2/types/capacity.types.ts | 4 +-
.../src/v2/types/container.types.ts | 56 +-
.../src/views/insights/insights.tsx | 12 +-
.../ozone/recon/api/TestClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/TestContainerEndpoint.java | 42 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 2 -
.../ozone/recon/api/TestExportJobManager.java | 432 +
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 2 -
.../ozone/recon/api/TestOpenContainerCount.java | 2 -
.../recon/api/TestOpenKeysSearchEndpoint.java | 2 -
.../recon/api/TestStorageDistributionEndpoint.java | 16 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 2 -
.../ozone/recon/api/types/TestExportJob.java | 110 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 799 +-
.../TestContainerHealthTaskRecordGenerator.java | 710 --
.../recon/fsck/TestReconReplicationManager.java | 515 +
.../ozone/recon/heatmap/TestHeatMapInfo.java | 2 -
.../recon/metrics/TestOzoneManagerSyncMetrics.java | 50 +
.../TestUnhealthyContainersDerbyPerformance.java | 765 ++
.../scm/TestReconStorageContainerSyncHelper.java | 185 +
...tOmTableInsightTaskStaleCounterAfterReinit.java | 243 +
...ntainersStateContainerIdIndexUpgradeAction.java | 117 +
hadoop-ozone/s3gateway/pom.xml | 7 +
.../hadoop/ozone/s3/AuthorizationFilter.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 5 +-
.../ozone/s3/commontypes/EncodingTypeObject.java | 5 +-
...er.java => AuditingBucketOperationHandler.java} | 48 +-
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 130 +-
.../ozone/s3/endpoint/BucketCrudHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 214 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 5 +
.../s3/endpoint/BucketOperationHandlerChain.java | 95 +
...CompleteMultipartUploadRequestUnmarshaller.java | 9 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 163 +-
.../s3/endpoint/ListMultipartUploadsHandler.java | 92 +
.../ozone/s3/endpoint/MessageUnmarshaller.java | 8 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 2 +
.../ozone/s3/endpoint/MultipartKeyHandler.java | 2 -
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 272 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 40 +-
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 9 +-
.../endpoint/PutBucketAclRequestUnmarshaller.java | 32 -
.../ozone/s3/endpoint/PutTaggingUnmarshaller.java | 29 -
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 86 +-
.../ozone/s3/endpoint/S3ConditionalRequest.java | 291 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 39 +-
.../hadoop/ozone/s3/exception/OSTSException.java | 21 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 311 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 9 +-
.../ozone/s3/signature/StringToSignProducer.java | 35 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 11 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 34 +-
.../apache/hadoop/ozone/s3sts/S3STSEndpoint.java | 70 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 73 +
.../hadoop/ozone/client/ObjectStoreStub.java | 54 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 95 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 16 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 5 +
.../ozone/s3/endpoint/EndpointTestUtils.java | 9 +-
.../ozone/s3/endpoint/TestBucketAclHandler.java | 82 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 3 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 88 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 85 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 236 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 8 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 32 +-
.../ozone/s3/exception/TestOS3Exceptions.java | 10 +-
.../ozone/s3/exception/TestOSTSExceptions.java | 8 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 9 +-
.../s3/signature/TestAWSSignatureProcessor.java | 20 +-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 41 +
hadoop-ozone/tools/pom.xml | 22 +-
.../ozone/local/LocalOzoneClusterConfig.java | 327 +
.../hadoop/ozone/local/LocalOzoneRuntime.java | 81 +
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 278 +
.../apache/hadoop/ozone/local}/package-info.java | 4 +-
.../ozone/local/TestLocalOzoneClusterConfig.java | 135 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 365 +
hadoop-ozone/vapor/pom.xml | 16 +-
.../containergenerator/GeneratorDatanode.java | 4 +-
pom.xml | 315 +-
.../FileSystem/failure_injector_fs.cc | 2 +-
781 files changed, 46599 insertions(+), 16085 deletions(-)
create mode 100644 .devcontainer/devcontainer.json
copy .github/workflows/{pull-request.yml => zizmor.yml} (64%)
copy hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/BooleanParam.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/LoopSampler.java (56%)
copy hadoop-hdds/{config/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java => common/src/main/java/org/apache/hadoop/hdds/tracing/SkipTracing.java} (85%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ipc_/WritableRpcEngine.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/security_/CustomizedCallbackHandler.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/security_/SaslMechanismFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/TestSecurityConfigTlsSettings.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestSpanSampling.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/io/TestRandomAccessFileChannel.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/ContainerChoosingPolicyFactory.java
rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/{DiskBalancerVolumeChoosingPolicy.java => ContainerCandidate.java} (53%)
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
delete mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultVolumeChoosingPolicy.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/TracingReconfigurationCallback.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/scm/ha/RetriableWithFailOverException.java => framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHandler.java} (72%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneBlacklist.java
copy hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/{TestSSLConnectionWithReload.java => TestGrpcTlsConfig.java} (50%)
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpServer2SSL.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIteratorWithDBClose.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{NullReplicationQueue.java => MonitoringReplicationQueue.java} (90%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/ReflectionUtil.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/ContainerStateManagerInvoker.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/DeletedBlockLogStateManagerInvoker.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/PipelineStateManagerInvoker.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/ScmInvoker.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/ScmInvokerCodeGenerator.java
copy hadoop-hdds/{cli-common/src/main/java/org/apache/hadoop/hdds/cli => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker}/package-info.java (84%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/PendingContainerTracker.java
copy hadoop-hdds/{framework/src/test/resources/ozone-site.xml => server-scm/src/main/resources/webapps/scm/ratis-events.html} (67%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerStateMap.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
copy hadoop-hdds/server-scm/src/{main/java/org/apache/hadoop/hdds/scm/ha/io/ScmCodec.java => test/java/org/apache/hadoop/hdds/scm/ha/io/TestScmListCodec.java} (51%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestPendingContainerTracker.java
create mode 100644 hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestContainerReportSuppressOptions.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/AuthorizationProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/ConfigProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/DiagnoseSubcommand.java
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetaStoreIterator.java => hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/DiagnosticProbe.java (66%)
copy hadoop-ozone/{s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java => cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/EnvironmentProbe.java} (57%)
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HostProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/HttpAuthProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/JvmKerberosProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/KerberosConfigProbe.java
copy hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/{ratis/RatisDebug.java => kerberos/KerberosSubcommand.java} (74%)
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/KerberosTicketProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/KeytabProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/KinitProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/PrincipalMappingProbe.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockmanagerMXBean.java => hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/ProbeResult.java (86%)
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/SecurityConfigProbe.java
create mode 100644 hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos/TranslatePrincipalSubcommand.java
copy {hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode => hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/kerberos}/package-info.java (87%)
create mode 100644 hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/kerberos/TestOzoneDebugKerberos.java
copy {hadoop-hdds/common/src/main/java/com/google/protobuf => hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/kerberos}/package-info.java (89%)
create mode 100644 hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/OzoneInteractiveShell.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartKey.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SubmitSnapshotDiffResponse.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcComposer.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/client/checksum/TestCrcUtil.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartPartInfo.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartPartKey.java
copy hadoop-ozone/common/src/{main => test}/java/org/apache/hadoop/ozone/security/acl/package-info.java (94%)
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSnapshotDiffResponse.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/snapshot/TestSubmitSnapshotDiffResponse.java
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-kerberos.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-defrag.robot
copy {hadoop-hdds/cli-common => hadoop-ozone/iceberg}/dev-support/findbugsExcludeFile.xml (100%)
copy {hadoop-hdds/erasurecode => hadoop-ozone/iceberg}/pom.xml (54%)
create mode 100644 hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java
create mode 100644 hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneUtils.java
copy {hadoop-hdds/common/src/main/java/com/google/protobuf => hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg}/package-info.java (90%)
create mode 100644 hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java
rename hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/{TestStorageDistributionEndpoint.java => AbstractTestStorageDistributionEndpoint.java} (60%)
create mode 100644 hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasksMultiNode.java
create mode 100644 hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpointEC.java
create mode 100644 hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpointRatis.java
copy hadoop-ozone/{integration-test => integration-test-s3}/src/test/resources/log4j.properties (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreamingDisabledDatanode.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerChoosingPolicy.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerPolicyPerformance.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{ServiceListJSONServlet.java => SnapshotListJSONServlet.java} (52%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMPeriodicMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java
copy hadoop-ozone/{httpfsgateway/src/main/java/org/apache/ozone/lib/service/Groups.java => ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManagerMXBean.java} (73%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html
copy hadoop-hdds/framework/src/test/resources/ozone-site.xml => hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html (67%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBCheckpointServletInodeBasedXferNonLeader.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListMultipartUploadsAcls.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListPartsAcls.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotListJSONServlet.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManagerMXBean.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneBlacklist.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ExportJobManager.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ExportJob.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/NoOpsContainerReplicaPendingOps.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManagerReport.java
delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ContainerHealthMetrics.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ContainerHealthTaskMetrics.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainersStateContainerIdIndexUpgradeAction.java
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/pagination.js
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/select/multiSelectMenuList.tsx
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestExportJobManager.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/types/TestExportJob.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestReconReplicationManager.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/metrics/TestOzoneManagerSyncMetrics.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUnhealthyContainersDerbyPerformance.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconStorageContainerSyncHelper.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTaskStaleCounterAfterReinit.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestUnhealthyContainersStateContainerIdIndexUpgradeAction.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/{AuditingObjectOperationHandler.java => AuditingBucketOperationHandler.java} (50%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationHandlerChain.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsHandler.java
delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketAclRequestUnmarshaller.java
delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutTaggingUnmarshaller.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java
copy hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestReconfigurableConfig.java => hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java (61%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
copy {hadoop-hdds/common/src/main/java/com/google/protobuf => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local}/package-info.java (90%)
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge remote-tracking branch 'upstream/master' into HDDS-13323-sts
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch HDDS-13323-sts
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 603a8abc3758bf93ad13b40cb71a11495cb8886f
Merge: 37a224b2176 99328c24239
Author: Fabian Morgan <fm...@gmail.com>
AuthorDate: Tue May 12 20:24:35 2026 -0700
Merge remote-tracking branch 'upstream/master' into HDDS-13323-sts
Conflicts:
hadoop-hdds/common/src/main/resources/ozone-default.xml
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/RequestContext.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketAclHandler.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
.devcontainer/devcontainer.json | 29 +
.github/dependabot.yml | 10 +
.github/pull_request_template.md | 3 +
.github/workflows/build-ratis.yml | 32 +-
.github/workflows/check.yml | 32 +-
.github/workflows/ci-with-ratis.yml | 9 +-
.github/workflows/ci.yml | 67 +-
.github/workflows/close-stale-prs.yaml | 4 +-
.github/workflows/generate-config-doc.yml | 11 +-
.github/workflows/intermittent-test-check.yml | 64 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 14 +-
.github/workflows/post-commit.yml | 7 +-
.github/workflows/pull-request.yml | 6 +-
.github/workflows/repeat-acceptance.yml | 22 +-
.github/workflows/schedule-label-pr.yml | 5 +-
.github/workflows/scheduled-cache-update.yml | 3 +-
.github/workflows/update-ozone-site-config-doc.yml | 45 +-
.github/workflows/{pull-request.yml => zizmor.yml} | 34 +-
.mvn/extensions.xml | 2 +-
CONTRIBUTING.md | 5 +-
README.md | 11 +-
SECURITY.md | 2 +-
dev-support/ci/xml_to_md.py | 31 +-
dev-support/pmd/pmd-ruleset.xml | 7 +
hadoop-hdds/cli-common/pom.xml | 16 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 33 +-
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 20 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 154 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 42 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 8 +
.../hdds/scm/storage/StreamBlockInputStream.java | 48 +-
.../io/ECBlockReconstructedStripeInputStream.java | 8 +-
.../scm/storage/TestStreamBlockInputStream.java | 193 +-
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 9 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 -
hadoop-hdds/common/pom.xml | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 22 +
.../org/apache/hadoop/hdds/client/BlockID.java | 4 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 10 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 37 +
.../balancer/ContainerBalancerConfiguration.java | 34 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hadoop/hdds/security/SecurityConfig.java | 36 +
.../apache/hadoop/hdds/tracing/LoopSampler.java} | 28 +-
.../apache/hadoop/hdds/tracing/SkipTracing.java | 18 +-
.../apache/hadoop/hdds/tracing/SpanSampler.java | 90 +
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 31 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 128 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 160 +-
.../apache/hadoop/hdds/utils/SlidingWindow.java | 11 +-
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 21 +-
.../hdds/utils/io/RandomAccessFileChannel.java | 37 +-
.../org/apache/hadoop/ipc_/AlignmentContext.java | 2 +-
.../src/main/java/org/apache/hadoop/ipc_/RPC.java | 18 +-
.../main/java/org/apache/hadoop/ipc_/Server.java | 51 +-
.../org/apache/hadoop/ipc_/WritableRpcEngine.java | 630 --
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 3 +
.../org/apache/hadoop/ozone/common/Checksum.java | 4 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 78 -
.../ozone/common/PureJavaCrc32ByteBuffer.java | 23 +-
.../ozone/common/PureJavaCrc32CByteBuffer.java | 23 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../security_/CustomizedCallbackHandler.java | 121 +
.../hadoop/security_/SaslMechanismFactory.java | 90 +
.../org/apache/hadoop/security_/SaslRpcClient.java | 16 +-
.../org/apache/hadoop/security_/SaslRpcServer.java | 63 +-
.../common/src/main/resources/ozone-default.xml | 230 +-
.../security/TestSecurityConfigTlsSettings.java | 90 +
.../hadoop/hdds/tracing/TestSpanSampling.java | 144 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 34 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 50 +-
.../hdds/utils/io/TestRandomAccessFileChannel.java | 153 +
.../ozone/common/TestChecksumByteBuffer.java | 4 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +-
hadoop-hdds/config/pom.xml | 10 +
.../hadoop/hdds/conf/ConfigFileAppender.java | 8 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 2 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 12 +-
.../ozone/container/common/impl/ContainerData.java | 48 +-
.../container/common/impl/ContainerDataYaml.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 237 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 6 +-
.../states/endpoint/VersionEndpointTask.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 13 +-
.../server/ratis/ContainerStateMachine.java | 22 +-
.../ozone/container/common/volume/HddsVolume.java | 44 +-
.../container/common/volume/MutableVolumeSet.java | 9 +-
.../container/common/volume/StorageVolume.java | 121 +-
.../common/volume/StorageVolumeChecker.java | 114 +-
.../common/volume/ThrottledAsyncChecker.java | 2 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ContainerChoosingPolicyFactory.java | 55 +
.../diskbalancer/DiskBalancerConfiguration.java | 138 +-
.../container/diskbalancer/DiskBalancerInfo.java | 55 +-
.../diskbalancer/DiskBalancerProtocolServer.java | 5 +-
.../diskbalancer/DiskBalancerService.java | 160 +-
.../container/diskbalancer/DiskBalancerYaml.java | 20 +-
...ChoosingPolicy.java => ContainerCandidate.java} | 42 +-
.../policy/ContainerChoosingPolicy.java | 36 +-
.../policy/DefaultContainerChoosingPolicy.java | 208 +-
.../policy/DefaultVolumeChoosingPolicy.java | 156 -
.../ECReconstructionCoordinator.java | 10 -
.../keyvalue/impl/FilePerBlockStrategy.java | 27 +-
.../ozone/container/metadata/AbstractRDBStore.java | 14 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 4 +-
.../container/replication/ReplicationServer.java | 5 +
.../ScmHAFinalizeUpgradeActionDatanode.java | 1 +
.../commands/ReplicateContainerCommand.java | 10 +-
.../container/common/impl/TestHddsDispatcher.java | 43 +
.../statemachine/TestDatanodeConfiguration.java | 32 +
.../server/ratis/TestContainerStateMachine.java | 45 +-
.../common/volume/TestStorageVolumeChecker.java | 185 +
.../volume/TestStorageVolumeHealthChecks.java | 149 +-
.../TestDefaultContainerChoosingPolicy.java | 780 +-
.../TestDefaultVolumeChoosingPolicy.java | 651 --
.../TestDiskBalancerProtocolServer.java | 65 +-
.../diskbalancer/TestDiskBalancerService.java | 81 +-
.../diskbalancer/TestDiskBalancerTask.java | 22 +-
.../diskbalancer/TestDiskBalancerYaml.java | 79 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 51 +
.../TestBackgroundContainerDataScanner.java | 114 +
hadoop-hdds/docs/content/design/diskbalancer.md | 16 +-
.../docs/content/design/s3-conditional-requests.md | 668 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 43 +-
.../docs/content/feature/DiskBalancer.zh.md | 25 +-
.../feature/Snapshot-Configuration-Properties.md | 1 +
hadoop-hdds/docs/content/feature/Snapshot.md | 9 +-
.../content/feature/SnapshotDefragmentation.md | 212 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 23 +-
hadoop-hdds/framework/pom.xml | 13 +-
.../hdds/conf/TracingReconfigurationCallback.java | 58 +
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 12 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 29 +
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHandler.java | 17 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 16 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 37 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 57 +-
.../symmetric/DefaultSecretKeySignerClient.java | 2 +-
.../hdds/security/symmetric/SecretKeyState.java | 9 +-
.../certificate/authority/CertificateStore.java | 8 +-
.../client/DefaultCertificateClient.java | 6 +-
.../certificate/utils/SelfSignedCertificate.java | 2 +-
.../apache/hadoop/hdds/server/OzoneBlacklist.java | 219 +
.../server/OzoneProtocolMessageDispatcher.java | 3 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 33 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 4 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 38 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 15 +-
.../apache/hadoop/hdds/utils/HttpServletUtils.java | 2 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 14 +-
.../hdds/utils/db/RDBStoreAbstractIterator.java | 19 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 4 +
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 23 +-
.../org/apache/hadoop/ozone/common/BlockGroup.java | 9 +-
.../apache/hadoop/ozone/common/DeletedBlock.java | 17 +-
.../src/main/resources/webapps/static/ozone.js | 2 +
.../resources/webapps/static/templates/menu.html | 2 +
.../hdds/fs/TestCachingSpaceUsageSource.java | 35 +
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 2 +-
.../hdds/security/ssl/TestGrpcTlsConfig.java | 276 +
.../hdds/server/http/TestHttpServer2SSL.java | 361 +
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 18 +-
.../hdds/utils/db/TestFixedLengthStringCodec.java | 7 +-
.../utils/db/TestRDBStoreIteratorWithDBClose.java | 308 +
.../src/main/proto/ScmAdminProtocol.proto | 28 +
.../src/main/proto/RpcHeader.proto | 2 +-
.../interface-client/src/main/proto/hdds.proto | 16 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../utils/db/managed/ManagedRocksIterator.java | 31 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 5 +
.../scm/block/DeletedBlockLogStateManager.java | 33 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 50 +-
.../container/AbstractContainerReportHandler.java | 26 +-
.../hdds/scm/container/ContainerManager.java | 39 +-
.../hdds/scm/container/ContainerManagerImpl.java | 40 +-
.../hdds/scm/container/ContainerStateManager.java | 41 +-
.../scm/container/ContainerStateManagerImpl.java | 44 +-
.../scm/container/balancer/ContainerBalancer.java | 20 +-
.../ContainerBalancerSelectionCriteria.java | 107 +-
.../container/balancer/ContainerBalancerTask.java | 2 +
.../hdds/scm/container/balancer/MoveManager.java | 109 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 -
.../container/placement/metrics/SCMMetrics.java | 36 +-
...nQueue.java => MonitoringReplicationQueue.java} | 4 +-
.../QuasiClosedStuckOverReplicationHandler.java | 4 +-
.../replication/QuasiClosedStuckReplicaCount.java | 77 +-
.../QuasiClosedStuckUnderReplicationHandler.java | 13 +-
.../replication/RatisUnderReplicationHandler.java | 9 +
.../container/replication/ReplicationManager.java | 74 +-
.../replication/health/EmptyContainerHandler.java | 74 +-
.../health/QuasiClosedStuckReplicationCheck.java | 20 +-
.../scm/container/states/ContainerStateMap.java | 14 +
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 +
.../apache/hadoop/hdds/scm/ha/ReflectionUtil.java | 67 -
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 10 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 35 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 36 +-
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 10 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 7 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 57 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 48 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 8 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 4 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 328 +
.../DeletedBlockLogStateManagerInvoker.java | 148 +
.../ha/invoker/PipelineStateManagerInvoker.java | 282 +
.../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 82 +
.../scm/ha/invoker/ScmInvokerCodeGenerator.java | 698 ++
.../hadoop/hdds/scm/ha/invoker/package-info.java | 13 +-
.../hadoop/hdds/scm/ha/io/ScmBigIntegerCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmBooleanCodec.java | 4 +-
.../hadoop/hdds/scm/ha/io/ScmByteStringCodec.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/io/ScmCodec.java | 3 +-
.../hadoop/hdds/scm/ha/io/ScmCodecFactory.java | 122 +-
.../apache/hadoop/hdds/scm/ha/io/ScmEnumCodec.java | 8 +-
.../hadoop/hdds/scm/ha/io/ScmIntegerCodec.java | 10 +-
.../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java | 84 +-
.../apache/hadoop/hdds/scm/ha/io/ScmLongCodec.java | 10 +-
.../hdds/scm/ha/io/ScmManagedSecretKeyCodec.java | 10 +-
.../scm/ha/io/ScmNonShadedByteStringCodec.java | 2 +-
.../ha/io/ScmNonShadedGeneratedMessageCodec.java | 10 +-
.../hadoop/hdds/scm/ha/io/ScmStringCodec.java | 2 +-
.../hdds/scm/ha/io/ScmX509CertificateCodec.java | 8 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 10 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 39 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 14 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 13 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hdds/scm/node/PendingContainerTracker.java | 225 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 30 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 24 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 -
.../hdds/scm/pipeline/PipelineStateManager.java | 14 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 5 +-
...inerLocationProtocolServerSideTranslatorPB.java | 53 +-
.../hdds/scm/security/RootCARotationHandler.java | 9 +-
.../scm/security/RootCARotationHandlerImpl.java | 4 +-
.../hdds/scm/security/RootCARotationManager.java | 17 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 109 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 20 +-
.../hdds/scm/server/StorageContainerManager.java | 27 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/server/StorageContainerManagerStarter.java | 2 -
.../server/upgrade/FinalizationStateManager.java | 9 +-
.../upgrade/FinalizationStateManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 5 +-
.../main/resources/webapps/scm/ratis-events.html} | 36 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../src/main/resources/webapps/scm/scm.js | 26 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 1 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 37 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../scm/container/TestContainerManagerImpl.java | 19 +-
.../scm/container/TestContainerReportHandler.java | 101 +-
.../scm/container/TestContainerStateManager.java | 44 +-
.../TestContainerBalancerSelectionCriteria.java | 188 +-
.../scm/container/balancer/TestMoveManager.java | 175 +
.../algorithms/TestContainerPlacementFactory.java | 2 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 24 +
...TestQuasiClosedStuckOverReplicationHandler.java | 96 +-
.../TestQuasiClosedStuckReplicaCount.java | 302 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 61 +-
.../health/TestEmptyContainerHandler.java | 36 +
.../TestQuasiClosedStuckReplicationCheck.java | 40 +-
.../container/states/TestContainerStateMap.java | 76 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 19 -
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 4 +-
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 +
.../hdds/scm/ha/io/TestScmBigIntegerCodec.java | 2 +-
.../hadoop/hdds/scm/ha/io/TestScmListCodec.java} | 30 +-
.../scm/ha/io/TestScmX509CertificateCodec.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 185 +
.../hdds/scm/node/TestPendingContainerTracker.java | 381 +
.../hdds/scm/node/states/TestNodeStateMap.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 11 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 56 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +-
.../scm/security/TestRootCARotationManager.java | 2 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 3 +-
.../scm/server/TestSCMClientProtocolServer.java | 31 +-
.../replicationManagerTests/quasi_closed.json | 57 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 6 +-
.../apache/ozone/test/TimedOutTestsListener.java | 4 +-
hadoop-ozone/cli-admin/pom.xml | 13 +
.../scm/cli/ContainerBalancerStatusSubcommand.java | 21 +-
.../hdds/scm/cli/ContainerOperationClient.java | 21 +
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 5 +-
.../scm/cli/container/ContainerIDParameters.java | 14 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 12 +-
.../hdds/scm/cli/container/ListSubcommand.java | 9 +-
.../scm/cli/container/ReconcileSubcommand.java | 26 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 65 +-
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 54 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 6 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 4 +-
.../cli/datanode/DiskBalancerReportSubcommand.java | 145 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 11 +
.../cli/datanode/DiskBalancerStatusSubcommand.java | 23 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 15 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 62 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 4 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 14 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 11 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 4 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 13 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 8 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 2 +-
.../ozone/admin/om/ListOpenFilesSubCommand.java | 61 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 18 +-
.../ozone/admin/scm/RotateKeySubCommand.java | 2 +-
.../TestContainerReportSuppressOptions.java | 310 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 5 +-
.../scm/cli/container/TestReconcileSubcommand.java | 50 +-
.../datanode/TestContainerBalancerSubCommand.java | 2 +-
.../cli/datanode/TestDiskBalancerSubCommands.java | 88 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 62 +-
hadoop-ozone/cli-debug/pom.xml | 20 +-
.../ozone/debug/audit/parser/AuditParser.java | 4 +-
.../debug/audit/parser/common/DatabaseHelper.java | 4 +-
.../audit/parser/handler/LoadCommandHandler.java | 6 +-
.../audit/parser/handler/QueryCommandHandler.java | 6 +-
.../parser/handler/TemplateCommandHandler.java | 16 +-
.../ozone/debug/kerberos/AuthorizationProbe.java | 106 +
.../hadoop/ozone/debug/kerberos/ConfigProbe.java | 116 +
.../ozone/debug/kerberos/DiagnoseSubcommand.java | 113 +
.../ozone/debug/kerberos/DiagnosticProbe.java} | 24 +-
.../ozone/debug/kerberos/EnvironmentProbe.java | 27 +-
.../hadoop/ozone/debug/kerberos/HostProbe.java | 72 +
.../hadoop/ozone/debug/kerberos/HttpAuthProbe.java | 52 +
.../ozone/debug/kerberos/JvmKerberosProbe.java | 98 +
.../ozone/debug/kerberos/KerberosConfigProbe.java | 61 +
.../ozone/debug/kerberos/KerberosSubcommand.java | 28 +-
.../ozone/debug/kerberos/KerberosTicketProbe.java | 95 +
.../hadoop/ozone/debug/kerberos/KeytabProbe.java | 93 +
.../hadoop/ozone/debug/kerberos/KinitProbe.java | 61 +
.../debug/kerberos/PrincipalMappingProbe.java | 113 +
.../hadoop/ozone/debug/kerberos/ProbeResult.java} | 14 +-
.../ozone/debug/kerberos/SecurityConfigProbe.java | 72 +
.../kerberos/TranslatePrincipalSubcommand.java | 78 +
.../hadoop/ozone/debug/kerberos/package-info.java} | 15 +-
.../debug/kerberos/TestOzoneDebugKerberos.java | 159 +
.../hadoop/ozone/debug/kerberos/package-info.java} | 12 +-
hadoop-ozone/cli-repair/pom.xml | 16 +-
.../schemaupgrade/ContainerUpgradeResult.java | 36 +-
.../schemaupgrade/VolumeUpgradeResult.java | 28 +-
hadoop-ozone/cli-shell/pom.xml | 16 +-
.../hadoop/ozone/shell/ListPaginationOptions.java | 2 +-
.../hadoop/ozone/shell/OzoneInteractiveShell.java | 84 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 26 +-
.../apache/hadoop/ozone/shell/acl/AclOption.java | 4 +-
.../hadoop/ozone/shell/bucket/BucketUri.java | 2 +-
.../ozone/shell/bucket/DeleteBucketHandler.java | 4 +-
.../hadoop/ozone/shell/common/VolumeBucketUri.java | 2 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 2 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 10 +-
.../hadoop/ozone/shell/prefix/PrefixUri.java | 2 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 43 +-
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 2 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 2 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +-
.../hadoop/ozone/shell/volume/VolumeUri.java | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 49 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 111 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 16 +
.../ozone/client/io/KeyDataStreamOutput.java | 6 +-
.../ozone/client/protocol/ClientProtocol.java | 138 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 249 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 9 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 36 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 4 +-
.../hadoop/ozone/client/checksum/CrcComposer.java | 79 +-
.../hadoop/ozone/client/checksum/CrcUtil.java | 155 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +-
.../hadoop/ozone/om/exceptions/OMException.java | 9 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 55 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 32 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 166 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 349 +
.../ozone/om/helpers/OmMultipartPartKey.java | 233 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 4 +-
.../ozone/om/helpers/OzoneFileStatusLight.java | 4 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 21 +
.../apache/hadoop/ozone/om/helpers/S3STSUtils.java | 16 +-
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 15 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 50 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 9 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 61 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 103 +-
.../ozone/security/acl/AssumeRoleRequest.java | 28 +-
.../hadoop/ozone/security/acl/RequestContext.java | 20 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 59 +-
.../ozone/snapshot/SubmitSnapshotDiffResponse.java | 65 +
.../org/apache/hadoop/ozone/util/RadixTree.java | 4 +-
.../ozone/client/checksum/TestCrcComposer.java | 222 +
.../hadoop/ozone/client/checksum/TestCrcUtil.java | 236 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 41 +
.../ozone/om/helpers/TestOmMultipartPartInfo.java | 143 +
.../ozone/om/helpers/TestOmMultipartPartKey.java | 204 +
.../om/helpers/TestOmSnapshotDiffJobCodec.java | 2 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 24 +
.../ozone/security/acl/TestAssumeRoleRequest.java | 73 +-
.../acl/iam/TestIamSessionPolicyResolver.java | 26 +-
.../hadoop/ozone/security/acl/package-info.java} | 12 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 78 +
.../snapshot/TestSubmitSnapshotDiffResponse.java | 97 +
hadoop-ozone/dist/README.md | 6 +-
hadoop-ozone/dist/pom.xml | 15 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +
.../dist/src/main/compose/common/hadoop-test.sh | 5 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 2 +-
.../src/main/compose/ozone-om-ha/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 4 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +-
.../compose/ozonesecure-ha/test-debug-tools.sh | 2 +
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 26 +
.../main/compose/upgrade/compose/ha/docker-config | 1 +
.../compose/upgrade/compose/non-ha/docker-config | 1 +
.../dist/src/main/compose/xcompat/new-cluster.yaml | 3 +
.../dist/src/main/compose/xcompat/old-cluster.yaml | 3 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 7 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 8 +-
.../dist/src/main/smoketest/cli/classpath.robot | 47 +-
.../src/main/smoketest/compatibility/setup.robot | 9 +-
.../src/main/smoketest/compatibility/write.robot | 65 +
.../smoketest/debug/ozone-debug-kerberos.robot | 59 +
.../omha/data-validation-after-om-bootstrap.robot | 15 +-
.../src/main/smoketest/s3/conditionalput.robot | 89 +
.../main/smoketest/scmha/container-create.robot | 2 +-
.../main/smoketest/snapshot/snapshot-defrag.robot | 172 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 12 +
.../dist/src/shell/ozone/ozone-functions.sh | 16 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 5 -
.../network-tests/src/test/compose/docker-config | 4 +-
hadoop-ozone/freon/pom.xml | 20 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 18 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 143 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 15 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 2 +-
hadoop-ozone/httpfsgateway/pom.xml | 32 -
.../src/main/resources/httpfs-default.xml | 8 -
.../iceberg/dev-support/findbugsExcludeFile.xml | 16 +
hadoop-ozone/iceberg/pom.xml | 105 +
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 463 +
.../ozone/iceberg/RewriteTablePathOzoneUtils.java | 126 +
.../apache/hadoop/ozone/iceberg/package-info.java} | 12 +-
.../iceberg/TestRewriteTablePathOzoneAction.java | 361 +
hadoop-ozone/integration-test-recon/pom.xml | 15 +
...> AbstractTestStorageDistributionEndpoint.java} | 282 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 1 +
.../ozone/recon/TestReconContainerEndpoint.java | 80 +-
.../ozone/recon/TestReconOmMetaManagerUtils.java | 33 +
.../apache/hadoop/ozone/recon/TestReconTasks.java | 802 +-
.../ozone/recon/TestReconTasksMultiNode.java | 178 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +
.../recon/TestStorageDistributionEndpointEC.java | 93 +
.../TestStorageDistributionEndpointRatis.java | 100 +
hadoop-ozone/integration-test-s3/pom.xml | 17 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 464 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 698 +-
.../src/test/resources/log4j.properties | 0
hadoop-ozone/integration-test/pom.xml | 25 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 20 +-
.../fs/contract/AbstractContractConcatTest.java | 8 +-
.../AbstractContractContentSummaryTest.java | 4 +-
.../AbstractContractCopyFromLocalTest.java | 10 +-
.../AbstractContractGetFileStatusTest.java | 10 +-
.../AbstractContractLeaseRecoveryTest.java | 14 +-
.../AbstractContractMultipartUploaderTest.java | 55 +-
.../fs/contract/AbstractContractOpenTest.java | 74 +-
.../AbstractContractRootDirectoryTest.java | 18 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 306 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 299 +-
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 323 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 144 +
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 51 +
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 1 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 8 +-
.../hdds/scm/TestStorageContainerManager.java | 11 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 27 +
.../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +-
.../TestReplicationManagerIntegration.java | 154 +
.../TestScmDataDistributionFinalization.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 8 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 333 +-
.../rpc/TestContainerStateMachineFailures.java | 65 +-
.../replication/TestContainerReplication.java | 53 +
.../TestContainerCommandReconciliation.java | 3 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 6 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 1 +
...stOzoneManagerHAFollowerReadWithAllRunning.java | 5 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 82 +
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 2 +
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../ozone/reconfig/TestOmReconfiguration.java | 74 +
.../ozone/reconfig/TestScmReconfiguration.java | 2 +
.../scm/node/TestContainerChoosingPolicy.java | 300 -
.../node/TestDiskBalancerPolicyPerformance.java | 381 +
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 -
.../ozone/shell/TestDeletedBlocksTxnShell.java | 3 +-
.../tools/contract/AbstractContractDistCpTest.java | 14 +-
.../src/test/resources/log4j.properties | 6 -
.../src/main/proto/OmClientProtocol.proto | 51 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +
hadoop-ozone/ozone-manager/pom.xml | 15 -
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 5 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 24 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 21 +
.../om/OMDBCheckpointServletInodeBasedXfer.java | 14 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 61 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 12 +-
.../hadoop/ozone/om/OmSnapshotLocalData.java | 2 +-
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 15 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 61 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 266 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 -
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 57 +-
.../hadoop/ozone/om/SnapshotListJSONServlet.java | 96 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 28 +-
.../hadoop/ozone/om/ha/OMPeriodicMetrics.java | 131 +
.../ozone/om/ratis/OzoneManagerStateMachine.java | 49 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 34 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 27 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 96 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 7 +
.../request/snapshot/OMSnapshotRenameRequest.java | 10 +
.../ozone/om/service/DirectoryDeletingService.java | 30 +-
.../ozone/om/service/KeyDeletingService.java | 8 +-
.../om/snapshot/OMSnapshotDirectoryMetrics.java | 328 +
.../om/snapshot/OmSnapshotLocalDataManager.java | 25 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 37 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 214 +-
.../om/snapshot/SnapshotDiffManagerMXBean.java} | 21 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 62 +-
.../hadoop/ozone/security/STSTokenIdentifier.java | 12 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 47 +
.../main/resources/webapps/ozoneManager/index.html | 4 +-
.../webapps/ozoneManager/om-snapshots.html | 176 +
.../resources/webapps/ozoneManager/ozoneManager.js | 155 +
.../webapps/ozoneManager/ratis-events.html | 36 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 80 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 2 +
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 50 +-
.../TestOzoneManagerListMultipartUploadsAcls.java | 196 +
.../ozone/om/TestOzoneManagerListPartsAcls.java | 196 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 55 +-
.../ozone/om/TestSnapshotListJSONServlet.java | 57 +
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 25 +
.../TestOmRatisSnapshotProvider.java | 8 +-
.../request/TestOMClientRequestWithUserInfo.java | 2 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +-
.../request/bucket/TestOMBucketCreateRequest.java | 35 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 77 +-
.../om/request/key/TestOMKeyCreateRequest.java | 235 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 +-
.../s3/security/TestS3GetSecretRequest.java | 2 +-
.../s3/tenant/TestOMTenantCreateRequest.java | 2 +-
.../s3/tenant/TestOMTenantDeleteRequest.java | 2 +-
.../snapshot/TestOMSnapshotRenameRequest.java | 20 +
.../validation/TestOMValidatorProcessor.java | 16 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +-
.../om/service/TestDirectoryDeletingService.java | 94 +-
.../ozone/om/service/TestKeyDeletingService.java | 37 +-
.../om/service/TestSnapshotDiffCleanupService.java | 2 +-
.../snapshot/TestOMSnapshotDirectoryMetrics.java | 125 +
.../snapshot/TestOmSnapshotLocalDataManager.java | 10 +
.../ozone/om/snapshot/TestSnapshotCache.java | 143 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 317 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 156 +
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 87 +
.../ozone/security/acl/TestOzoneBlacklist.java | 235 +
.../ozone/security/acl/TestRequestContext.java | 6 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 13 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 61 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 67 +-
hadoop-ozone/pom.xml | 9 +
.../ozone/recon/codegen/TableNamingStrategy.java | 12 +-
.../recon/schema/ContainerSchemaDefinition.java | 20 +-
hadoop-ozone/recon/pom.xml | 14 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 71 +
.../ozone/recon/api/ClusterStateEndpoint.java | 7 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 268 +-
.../hadoop/ozone/recon/api/ExportJobManager.java | 404 +
.../recon/api/StorageDistributionEndpoint.java | 69 +-
.../ozone/recon/api/handlers/BucketHandler.java | 14 +-
.../hadoop/ozone/recon/api/types/ExportJob.java | 222 +
.../ozone/recon/api/types/GlobalStorageReport.java | 20 +-
.../ozone/recon/api/types/UsedSpaceBreakDown.java | 12 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 793 +-
.../fsck/NoOpsContainerReplicaPendingOps.java | 112 +
.../ozone/recon/fsck/ReconReplicationManager.java | 661 ++
.../recon/fsck/ReconReplicationManagerReport.java | 66 +
.../recon/metrics/ContainerHealthMetrics.java | 86 -
.../recon/metrics/ContainerHealthTaskMetrics.java | 75 +
.../recon/metrics/OzoneManagerSyncMetrics.java | 2 +-
.../persistence/ContainerHealthSchemaManager.java | 596 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 128 +-
.../recon/scm/ReconStorageContainerSyncHelper.java | 113 +
.../recon/spi/StorageContainerServiceProvider.java | 8 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 9 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 16 +-
.../impl/StorageContainerServiceProviderImpl.java | 9 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 3 +-
.../ozone/recon/upgrade/ReconLayoutFeature.java | 4 +-
...ntainersStateContainerIdIndexUpgradeAction.java | 86 +
.../webapps/recon/ozone-recon-web/.eslintrc.json | 180 +-
.../webapps/recon/ozone-recon-web/api/db.json | 2736 ++++-
.../recon/ozone-recon-web/api/pagination.js | 86 +
.../webapps/recon/ozone-recon-web/api/routes.json | 5 +
.../webapps/recon/ozone-recon-web/package.json | 10 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 10840 +++++++++++--------
.../mocks/capacityMocks/capacityResponseMocks.ts | 4 +-
.../src/v2/components/plots/insightsFilePlot.tsx | 18 +-
.../src/v2/components/select/multiSelect.tsx | 134 +-
.../v2/components/select/multiSelectMenuList.tsx | 160 +
.../src/v2/components/select/singleSelect.tsx | 1 +
.../src/v2/components/tables/bucketsTable.tsx | 2 +
.../src/v2/components/tables/containersTable.tsx | 71 +-
.../src/v2/components/tables/datanodesTable.tsx | 1 +
.../src/v2/components/tables/pipelinesTable.tsx | 2 +-
.../src/v2/constants/capacity.constants.tsx | 4 +-
.../src/v2/constants/select.constants.tsx | 7 +-
.../src/v2/pages/buckets/buckets.tsx | 9 +-
.../src/v2/pages/capacity/capacity.tsx | 4 +-
.../src/v2/pages/containers/containers.tsx | 816 +-
.../src/v2/pages/insights/insights.tsx | 1 +
.../ozone-recon-web/src/v2/types/capacity.types.ts | 4 +-
.../src/v2/types/container.types.ts | 56 +-
.../src/views/insights/insights.tsx | 12 +-
.../ozone/recon/api/TestClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/TestContainerEndpoint.java | 42 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 2 -
.../ozone/recon/api/TestExportJobManager.java | 432 +
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 2 -
.../ozone/recon/api/TestOpenContainerCount.java | 2 -
.../recon/api/TestOpenKeysSearchEndpoint.java | 2 -
.../recon/api/TestStorageDistributionEndpoint.java | 16 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 2 -
.../ozone/recon/api/types/TestExportJob.java | 110 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 799 +-
.../TestContainerHealthTaskRecordGenerator.java | 710 --
.../recon/fsck/TestReconReplicationManager.java | 515 +
.../ozone/recon/heatmap/TestHeatMapInfo.java | 2 -
.../recon/metrics/TestOzoneManagerSyncMetrics.java | 50 +
.../TestUnhealthyContainersDerbyPerformance.java | 765 ++
.../scm/TestReconStorageContainerSyncHelper.java | 185 +
...tOmTableInsightTaskStaleCounterAfterReinit.java | 243 +
...ntainersStateContainerIdIndexUpgradeAction.java | 117 +
hadoop-ozone/s3gateway/pom.xml | 7 +
.../hadoop/ozone/s3/AuthorizationFilter.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 5 +-
.../ozone/s3/commontypes/EncodingTypeObject.java | 5 +-
.../endpoint/AuditingBucketOperationHandler.java | 70 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 130 +-
.../ozone/s3/endpoint/BucketCrudHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 214 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 5 +
.../s3/endpoint/BucketOperationHandlerChain.java | 95 +
...CompleteMultipartUploadRequestUnmarshaller.java | 9 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 163 +-
.../s3/endpoint/ListMultipartUploadsHandler.java | 92 +
.../ozone/s3/endpoint/MessageUnmarshaller.java | 8 +-
.../endpoint/MultiDeleteRequestUnmarshaller.java | 2 +
.../ozone/s3/endpoint/MultipartKeyHandler.java | 2 -
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 272 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 40 +-
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 9 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 86 +-
.../ozone/s3/endpoint/S3ConditionalRequest.java | 291 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 39 +-
.../hadoop/ozone/s3/exception/OSTSException.java | 21 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 311 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 9 +-
.../ozone/s3/signature/StringToSignProducer.java | 35 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 11 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 34 +-
.../apache/hadoop/ozone/s3sts/S3STSEndpoint.java | 70 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 73 +
.../hadoop/ozone/client/ObjectStoreStub.java | 54 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 95 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 16 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 5 +
.../ozone/s3/endpoint/EndpointTestUtils.java | 9 +-
.../ozone/s3/endpoint/TestBucketAclHandler.java | 82 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 3 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 88 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 85 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 236 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 8 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 32 +-
.../ozone/s3/exception/TestOS3Exceptions.java | 10 +-
.../ozone/s3/exception/TestOSTSExceptions.java | 8 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 9 +-
.../s3/signature/TestAWSSignatureProcessor.java | 23 +-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 41 +
hadoop-ozone/tools/pom.xml | 22 +-
.../ozone/local/LocalOzoneClusterConfig.java | 327 +
.../hadoop/ozone/local/LocalOzoneRuntime.java | 81 +
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 278 +
.../apache/hadoop/ozone/local/package-info.java} | 12 +-
.../ozone/local/TestLocalOzoneClusterConfig.java | 135 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 365 +
hadoop-ozone/vapor/pom.xml | 16 +-
.../containergenerator/GeneratorDatanode.java | 4 +-
pom.xml | 315 +-
.../FileSystem/failure_injector_fs.cc | 2 +-
779 files changed, 46940 insertions(+), 15928 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index 2066a99baa4,f554b440602..e844447cd4b
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -5051,25 -5068,16 +5116,36 @@@
<value>5m</value>
<description>Interval for cleaning up orphan snapshot local data versions corresponding to snapshots</description>
</property>
-
+ <property>
+ <name>ozone.om.sts.token.cleanup.service.interval</name>
+ <value>3h</value>
+ <tag>OZONE, OM, PERFORMANCE, SECURITY</tag>
+ <description>
+ A background job that periodically checks revoked STS token entries and
+ deletes ones that have existed for 12 hours. This entry controls the interval of this
+ cleanup check. Unit could be defined with postfix (ns,ms,s,m,h,d).
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.sts.token.cleanup.service.timeout</name>
+ <value>15m</value>
+ <tag>OZONE, OM, PERFORMANCE, SECURITY</tag>
+ <description>
+ A timeout value for the revoked STS token cleanup service. If this is set
+ greater than 0, the service will stop waiting for the deletion
+ completion after this time. Unit could be defined with postfix (ns,ms,s,m,h,d).
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, SCM</tag>
+ <description>The maximum number of events that can be pending in SCM Ratis.</description>
+ </property>
+ <property>
+ <name>ozone.om.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, OM</tag>
+ <description>The maximum number of events that can be pending in OM Ratis.</description>
+ </property>
</configuration>
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 445355987d7,ea17ad9ca85..0a2d506fd69
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -687,16 -699,10 +699,20 @@@ public final class OMConfigKeys
"ozone.om.snapshot.local.data.manager.service.interval";
public static final String OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT = "5m";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_INTERVAL
+ = "ozone.om.sts.token.cleanup.service.interval";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_INTERVAL_DEFAULT
+ = "3h";
+
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_TIMEOUT
+ = "ozone.om.sts.token.cleanup.service.timeout";
+ public static final String OZONE_OM_STS_TOKEN_CLEANUP_SERVICE_TIMEOUT_DEFAULT
+ = "15m";
+
+ public static final String OZONE_OM_RATIS_EVENTS_MAX_LIMIT =
+ "ozone.om.ratis.events.max.limit";
+ public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
+
/**
* Never constructed.
*/
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index fcd1e7807f1,240e99e7d67..39e1042b474
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -276,7 -276,10 +276,12 @@@ public class OMException extends IOExce
TOO_MANY_SNAPSHOTS,
+ ETAG_MISMATCH,
+
+ ETAG_NOT_AVAILABLE,
+
+ ATOMIC_WRITE_CONFLICT,
+ REVOKED_TOKEN,
+ MALFORMED_POLICY_DOCUMENT,
}
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index b5cdcc52540,5d2de09c48e..3d29637d973
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@@ -61,9 -61,7 +61,10 @@@ public final class OmKeyArgs extends Wi
// This allows a key to be created an committed atomically if the original has not
// been modified.
private Long expectedDataGeneration = null;
+ // Original S3/list prefix when keyName is empty (root listing). Used for STS
+ // auth to check LIST on this prefix instead of "*".
+ private final String listPrefix;
+ private final String expectedETag;
private OmKeyArgs(Builder b) {
super(b);
@@@ -85,7 -83,7 +86,8 @@@
this.ownerName = b.ownerName;
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
+ this.listPrefix = b.listPrefix;
+ this.expectedETag = b.expectedETag;
}
public boolean getIsMultipartKey() {
@@@ -168,14 -166,10 +170,18 @@@
return expectedDataGeneration;
}
+ /**
+ * Original S3/list prefix when keyName is empty (root listing).
+ * Used for STS auth to check LIST on this prefix instead of "*".
+ */
+ public String getListPrefix() {
+ return listPrefix;
+ }
+
+ public String getExpectedETag() {
+ return expectedETag;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@@ -246,7 -243,7 +255,8 @@@
private boolean forceUpdateContainerCacheFromSCM;
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
+ private String listPrefix = null;
+ private String expectedETag;
public Builder() {
this(AclListBuilder.empty());
@@@ -276,9 -289,9 +302,10 @@@
this.forceUpdateContainerCacheFromSCM =
obj.forceUpdateContainerCacheFromSCM;
this.expectedDataGeneration = obj.expectedDataGeneration;
+ this.expectedETag = obj.expectedETag;
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
+ this.listPrefix = obj.listPrefix;
}
public Builder setVolumeName(String volume) {
@@@ -412,11 -425,11 +439,16 @@@
return this;
}
+ public Builder setListPrefix(String prefix) {
+ this.listPrefix = prefix;
+ return this;
+ }
+
+ public Builder setExpectedETag(String eTag) {
+ this.expectedETag = eTag;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(this);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
index c70c01a8723,00000000000..763c8fe9bfa
mode 100644,000000..100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
@@@ -1,170 -1,0 +1,170 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+
+import com.google.common.base.Strings;
+import java.util.Map;
+import net.jcip.annotations.Immutable;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+
+/**
+ * Utility class containing constants and validation methods shared by STS endpoint and OzoneManager processing.
+ */
+@Immutable
+public final class S3STSUtils {
+ // STS API constants
+ public static final int DEFAULT_DURATION_SECONDS = 3600; // 1 hour
+ public static final int MAX_DURATION_SECONDS = 43200; // 12 hours
+ public static final int MIN_DURATION_SECONDS = 900; // 15 minutes
+
+ public static final int ASSUME_ROLE_SESSION_NAME_MIN_LENGTH = 2;
+ public static final int ASSUME_ROLE_SESSION_NAME_MAX_LENGTH = 64;
+
+ // AWS limit for session policy is 2048 characters
+ public static final int MAX_SESSION_POLICY_LENGTH = 2048;
+
+ private S3STSUtils() {
+ }
+
+ /**
+ * Adds standard AssumeRole audit params.
+ */
+ public static void addAssumeRoleAuditParams(Map<String, String> auditParams, String roleArn, String roleSessionName,
+ String awsIamSessionPolicy, int duration, String requestId) {
+
+ auditParams.put("action", "AssumeRole");
+ auditParams.put("roleArn", roleArn);
+ auditParams.put("roleSessionName", roleSessionName);
+ auditParams.put("duration", String.valueOf(duration));
+ auditParams.put("isPolicyIncluded", Strings.isNullOrEmpty(awsIamSessionPolicy) ? "N" : "Y");
+ auditParams.put("requestId", requestId);
+ }
+
+ /**
+ * Validates the duration in seconds.
+ * @param durationSeconds duration in seconds
+ * @return validated duration
+ * @throws OMException if duration is invalid
+ */
+ public static int validateDuration(Integer durationSeconds) throws OMException {
+ if (durationSeconds == null) {
+ return DEFAULT_DURATION_SECONDS;
+ }
+
+ if (durationSeconds < MIN_DURATION_SECONDS || durationSeconds > MAX_DURATION_SECONDS) {
+ throw new OMException(
+ "Invalid Value: DurationSeconds must be between " + MIN_DURATION_SECONDS + " and " + MAX_DURATION_SECONDS +
+ " seconds", INVALID_REQUEST);
+ }
+
+ return durationSeconds;
+ }
+
+ /**
+ * Validates the role session name.
+ * @param roleSessionName role session name
+ * @throws OMException if role session name is invalid
+ */
+ public static void validateRoleSessionName(String roleSessionName) throws OMException {
+ if (Strings.isNullOrEmpty(roleSessionName)) {
+ throw new OMException(
+ "Value null at 'roleSessionName' failed to satisfy constraint: Member must not be null", INVALID_REQUEST);
+ }
+
+ final int roleSessionNameLength = roleSessionName.length();
+ if (roleSessionNameLength < ASSUME_ROLE_SESSION_NAME_MIN_LENGTH ||
+ roleSessionNameLength > ASSUME_ROLE_SESSION_NAME_MAX_LENGTH) {
+ throw new OMException("Invalid RoleSessionName length " + roleSessionNameLength + ": it must be " +
+ ASSUME_ROLE_SESSION_NAME_MIN_LENGTH + "-" + ASSUME_ROLE_SESSION_NAME_MAX_LENGTH + " characters long and " +
+ "contain only alphanumeric characters and +, =, ,, ., @, -", INVALID_REQUEST);
+ }
+
+ // AWS allows: alphanumeric, +, =, ,, ., @, -
+ // Pattern: [\w+=,.@-]*
+ // Don't use regex for performance reasons
+ for (int i = 0; i < roleSessionNameLength; i++) {
+ final char c = roleSessionName.charAt(i);
+ if (!isRoleSessionNameChar(c)) {
+ throw new OMException("Invalid character '" + c + "' in RoleSessionName: it must be " +
+ ASSUME_ROLE_SESSION_NAME_MIN_LENGTH + "-" + ASSUME_ROLE_SESSION_NAME_MAX_LENGTH + " characters long and " +
+ "contain only alphanumeric characters and +, =, ,, ., @, -", INVALID_REQUEST);
+ }
+ }
+ }
+
+ private static boolean isRoleSessionNameChar(char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
+ c == '_' || c == '+' || c == '=' || c == ',' || c == '.' || c == '@' || c == '-';
+ }
+
+ /**
+ * Validates the session policy length.
+ * @param awsIamSessionPolicy session policy
+ * @throws OMException if policy length is invalid
+ */
+ public static void validateSessionPolicy(String awsIamSessionPolicy) throws OMException {
+ if (awsIamSessionPolicy != null && awsIamSessionPolicy.length() > MAX_SESSION_POLICY_LENGTH) {
+ throw new OMException(
+ "Value '" + awsIamSessionPolicy + "' at 'policy' failed to satisfy constraint: Member " +
+ "must have length less than or equal to " + MAX_SESSION_POLICY_LENGTH, INVALID_REQUEST);
+ }
+ }
+
+ /**
+ * Generates the assumed role user ARN.
+ * @param validRoleArn valid role ARN
+ * @param roleSessionName role session name
+ * @return assumed role user ARN
+ */
+ public static String toAssumedRoleUserArn(String validRoleArn, String roleSessionName) {
+ // We already know the roleArn is valid, so perform the conversion for assumed role user arn format
+ // RoleArn format: arn:aws:iam::<account-id>:role/<role-name>
+ // Assumed role user arn format: arn:aws:sts::<account-id>:assumed-role/<role-name>/<role-session-name>
+ final String[] parts = splitRoleArnWithoutRegex(validRoleArn);
+
+ final String partition = parts[1];
+ final String accountId = parts[4];
+ final String resource = parts[5];
+ final String roleName = resource.substring("role/".length());
+
+ //noinspection StringBufferReplaceableByString
- final StringBuilder stringBuilder = new StringBuilder("arn:");
- stringBuilder.append(partition);
- stringBuilder.append(":sts::");
- stringBuilder.append(accountId);
- stringBuilder.append(":assumed-role/");
- stringBuilder.append(roleName);
- stringBuilder.append('/');
- stringBuilder.append(roleSessionName);
++ final StringBuilder stringBuilder = new StringBuilder("arn:")
++ .append(partition)
++ .append(":sts::")
++ .append(accountId)
++ .append(":assumed-role/")
++ .append(roleName)
++ .append('/')
++ .append(roleSessionName);
+ return stringBuilder.toString();
+ }
+
+ private static String[] splitRoleArnWithoutRegex(String roleArn) {
+ final String[] parts = new String[6];
+ int start = 0;
+ for (int i = 0; i < 5; i++) {
+ final int end = roleArn.indexOf(':', start);
+ parts[i] = roleArn.substring(start, end);
+ start = end + 1;
+ }
+ parts[5] = roleArn.substring(start);
+ return parts;
+ }
+}
diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
index 1d976514632,00000000000..f8d37a113d4
mode 100644,000000..100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
@@@ -1,2256 -1,0 +1,2256 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security.acl.iam;
+
+import static java.util.Collections.emptySet;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
+import static org.apache.hadoop.ozone.security.acl.AssumeRoleRequest.OzoneGrant;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.CREATE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.DELETE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.LIST;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ_ACL;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE_ACL;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.NATIVE;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.AuthorizerType.RANGER;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3ResourceType;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.buildCaseInsensitiveS3ActionMap;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.createPathsAndPermissions;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.groupObjectsByAcls;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.mapPolicyActionsToS3Actions;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.resolve;
+import static org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.validateAndCategorizeResources;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.security.acl.IOzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver.S3Action;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for {@link IamSessionPolicyResolver}.
+ * */
+public class TestIamSessionPolicyResolver {
+
+ private static final String VOLUME = "s3v";
+
+ @Test
+ public void testUnsupportedConditionOperatorThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringNotEqualsIgnoreCase\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Condition operator - StringNotEqualsIgnoreCase",
+ NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testUnsupportedConditionAttributeThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"aws:SourceArn\": \"arn:aws:s3:::d\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Condition key name - aws:SourceArn", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testUnsupportedEffectThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Deny\",\n" + // unsupported effect
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::proj-*\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Effect - Deny", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testInvalidJsonWithoutStatementThrows() {
+ final String json = "{\n" +
+ " \"RandomAttribute\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid policy JSON - missing Statement", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidEffectThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": [\"Allow\"],\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Effect in JSON policy (must be a String) - [\"Allow\"]",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testMissingEffectInStatementThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Effect is missing from JSON policy", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidNumberOfConditionsThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": [\n" +
+ " {\n" +
+ " \"StringEquals\": {\n" +
+ " \"aws:SourceArn\": \"arn:aws:s3:::d\"\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"StringEquals\": {\n" +
+ " \"aws:SourceArn\": \"arn:aws:s3:::e\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Only one Condition is supported", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testInvalidConditionThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": [\"RandomCondition\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Condition (must have operator StringEquals or StringLike and key name " +
+ "s3:prefix) - [\"RandomCondition\"]", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidConditionAttributeMissingStringEqualsThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": null }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Missing Condition operator value for StringEquals",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidConditionAttributeStructureThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": [{ \"s3:prefix\": \"folder/\" }] }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Condition operator value structure - [{\"s3:prefix\":\"folder/\"}]",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testInvalidJsonThrows() {
+ final String invalidJson = "{[{{}]\"\"";
+
+ expectResolveThrowsForBothAuthorizers(
+ invalidJson, "IAM session policy: Invalid policy JSON (most likely JSON structure is incorrect)",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testJsonExceedsMaxLengthThrows() {
+ final String json = createJsonStringLargerThan2048Characters();
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid policy JSON - exceeds maximum length of 2048 characters",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testJsonAtMaxLengthSucceeds() throws OMException {
+ // Create a JSON string that is exactly 2048 characters
+ final String json = create2048CharJsonString();
+ assertThat(json.length()).isEqualTo(2048);
+
+ // Must not throw an exception
+ resolve(json, VOLUME, NATIVE);
+ resolve(json, VOLUME, RANGER);
+ }
+
+ @Test
+ public void testConditionKeyMustBeCaseInsensitive() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"S3:PRefiX\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ // Must not throw exception
+ resolve(json, VOLUME, NATIVE);
+ resolve(json, VOLUME, RANGER);
+ }
+
+ @Test
+ public void testEffectMustBeCaseSensitive() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"aLLOw\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::b\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": \"x/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Effect - aLLOw", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMapMatchesConstant() {
+ assertThat(buildCaseInsensitiveS3ActionMap()).isEqualTo(IamSessionPolicyResolver.S3_ACTION_MAP_CI);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMap() {
+ final Map<String, Set<S3Action>> caseInsensitiveS3ActionMap = buildCaseInsensitiveS3ActionMap();
+ // Verify that individual S3 actions are present
+ assertThat(caseInsensitiveS3ActionMap).containsKeys(
+ "s3:listbucket", "s3:getobject", "s3:putobject", "s3:deleteobject", "s3:createbucket", "s3:listallmybuckets");
+
+ // Verify that wildcard actions are present
+ assertThat(caseInsensitiveS3ActionMap).containsKeys(
+ "s3:*", "s3:get*", "s3:put*", "s3:list*", "s3:delete*", "s3:create*");
+
+ // Verify s3:Get* contains Get actions
+ final Set<S3Action> getActions = caseInsensitiveS3ActionMap.get("s3:get*");
+ assertThat(getActions).containsOnly(
+ S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+
+ // Verify s3:Put* contains Put actions
+ final Set<S3Action> putActions = caseInsensitiveS3ActionMap.get("s3:put*");
+ assertThat(putActions).containsOnly(
+ S3Action.PUT_OBJECT, S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL);
+
+ // Verify s3:List* contains List actions
+ final Set<S3Action> listActions = caseInsensitiveS3ActionMap.get("s3:list*");
+ assertThat(listActions).containsOnly(
+ S3Action.LIST_BUCKET, S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET_MULTIPART_UPLOADS,
+ S3Action.LIST_MULTIPART_UPLOAD_PARTS);
+
+ // Verify s3:Delete* contains Delete actions
+ final Set<S3Action> deleteActions = caseInsensitiveS3ActionMap.get("s3:delete*");
+ assertThat(deleteActions).containsOnly(
+ S3Action.DELETE_OBJECT, S3Action.DELETE_BUCKET, S3Action.DELETE_OBJECT_TAGGING);
+
+ // Verify s3:Create* contains Create actions
+ final Set<S3Action> createActions = caseInsensitiveS3ActionMap.get("s3:create*");
+ assertThat(createActions).containsOnly(S3Action.CREATE_BUCKET);
+ }
+
+ @Test
+ public void testBuildCaseInsensitiveS3ActionMapIndividualActionsContainSingleEntry() {
+ final Map<String, Set<S3Action>> actionMap = buildCaseInsensitiveS3ActionMap();
+
+ // Individual actions should map to a set with exactly one entry
+ final Set<S3Action> listBucketAction = actionMap.get("s3:listbucket");
+ assertThat(listBucketAction).hasSize(1);
+
+ final Set<S3Action> getObjectAction = actionMap.get("s3:getobject");
+ assertThat(getObjectAction).hasSize(1);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithNullReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(null);
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithEmptyListReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(emptySet());
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithSingleActionMapsCorrectly() {
+ final Set<S3Action> listBucket = mapPolicyActionsToS3Actions(Collections.singleton("s3:ListBucket"));
+ assertThat(listBucket).containsOnly(S3Action.LIST_BUCKET);
+
+ // Ensure case-insensitive action works
+ final Set<S3Action> listBucketCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:ListBuCKet"));
+ assertThat(listBucketCi).containsOnly(S3Action.LIST_BUCKET);
+
+ final Set<S3Action> deleteObject = mapPolicyActionsToS3Actions(Collections.singleton("s3:DeleteObject"));
+ assertThat(deleteObject).containsOnly(S3Action.DELETE_OBJECT);
+
+ // Ensure case-insensitive action works
+ final Set<S3Action> deleteObjectCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:DeLETeObjeCT"));
+ assertThat(deleteObjectCi).containsOnly(S3Action.DELETE_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithMultipleActionsMapAllCorrectly() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:ListBucket", "s3:GetObject", "s3:PutObject"));
+ assertThat(result).containsOnly(S3Action.LIST_BUCKET, S3Action.GET_OBJECT, S3Action.PUT_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithWildcardExpansion() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(Collections.singleton("s3:Get*"));
+ assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+
+ // Ensure it is case-insensitive
+ final Set<S3Action> resultCi = mapPolicyActionsToS3Actions(Collections.singleton("s3:gET*"));
+ assertThat(resultCi).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithS3StarReturnsAll() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(Collections.singleton("s3:*"));
+ assertThat(result).containsOnly(S3Action.ALL_S3);
+
+ final Set<S3Action> resultCi = mapPolicyActionsToS3Actions(Collections.singleton("S3:*"));
+ assertThat(resultCi).containsOnly(S3Action.ALL_S3);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsIgnoresUnsupportedActions() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:GetAccelerateConfiguration", "s3:GetObject"));
+ // Unsupported action should be silently ignored
+ assertThat(result).containsOnly(S3Action.GET_OBJECT);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithOnlyUnsupportedActionsReturnsEmpty() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(
+ strSet("s3:GetAccelerateConfiguration", "s3:PutBucketVersioning"));
+ assertThat(result).isEmpty();
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsDeduplicatesResults() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:GetObject", "s3:GetBucketAcl"));
+ assertThat(result).containsOnly(S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsHandlesMultipleWildcards() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:Get*", "s3:Put*"));
+ assertThat(result).containsOnly(
+ S3Action.GET_OBJECT, S3Action.GET_BUCKET_ACL, S3Action.GET_OBJECT_TAGGING, S3Action.PUT_OBJECT,
+ S3Action.PUT_OBJECT_TAGGING, S3Action.PUT_BUCKET_ACL);
+ }
+
+ @Test
+ public void testMapPolicyActionsToS3ActionsWithS3StarIgnoresOtherActions() {
+ final Set<S3Action> result = mapPolicyActionsToS3Actions(strSet("s3:*", "s3:GetObject", "s3:PutObject"));
+ // When s3:* is present, it should return only the ALL_S3 action
+ assertThat(result).containsOnly(S3Action.ALL_S3);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithWildcard() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("*"));
+ assertThat(resultRanger).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.ANY, "*", null, null));
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithSingleBucket() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.BUCKET, "my-bucket", null, null);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::my-bucket"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::my-bucket"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcard() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.BUCKET_WILDCARD, "my-bucket*", null, null);
+
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::my-bucket*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::my-bucket*"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcardAndExactObjectKey() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "*", null, "myKey.txt");
+
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::*/myKey.txt")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::*/myKey.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketWildcardAndObjectWildcard() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::*/*")),
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer",
+ NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "*", "*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::*/*"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndExactObjectKey() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt");
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket1/key.txt"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket1/key.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndExactObjectKeyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_EXACT, "bucket2", null, "path/folder/nested/key.txt");
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket2/path/folder/nested/key.txt"));
+ assertThat(resultNative).containsOnly(expectedResourceSpec);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket2/path/folder/nested/key.txt"));
+ assertThat(resultRanger).containsOnly(expectedResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndEmpty() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndEmptyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "path/b/", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/path/b/*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "path/b/*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/path/b/*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndNonEmpty() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket3", "test", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket3/test*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "test*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/test*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixAndNonEmptyWithPath() throws OMException {
+ final IamSessionPolicyResolver.ResourceSpec expectedNativeResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX, "bucket", "a/b/test", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, Collections.singleton("arn:aws:s3:::bucket/a/b/test*"));
+ assertThat(resultNative).containsOnly(expectedNativeResourceSpec);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket", "a/b/test*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket/a/b/test*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardNotAtEnd() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*.log")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*.log", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*.log"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardNotAtEndWithPath() throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket/a/q/*.ps")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer if " +
+ "wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket", "a/q/*.ps", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket/a/q/*.ps"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardOneAtEndAndOneNotAtEnd()
+ throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/*key*")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "*key*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/*key*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithBucketAndObjectPrefixWildcardOneAtEndAndOneNotAtEndWithPath()
+ throws OMException {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::bucket3/a/b/t/*key*")),
+ "IAM session policy: Wildcard prefix patterns are not supported for Ozone native authorizer " +
+ "if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final IamSessionPolicyResolver.ResourceSpec expectedRangerResourceSpec = new IamSessionPolicyResolver.ResourceSpec(
+ S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket3", "a/b/t/*key*", null);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, Collections.singleton("arn:aws:s3:::bucket3/a/b/t/*key*"));
+ assertThat(resultRanger).containsOnly(expectedRangerResourceSpec);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithMultipleResources() throws OMException {
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultNative = validateAndCategorizeResources(
+ NATIVE, strSet("arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2/*", "arn:aws:s3:::bucket3/key.txt"));
+ assertThat(resultNative).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket2", "", null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket3", null, "key.txt"));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resultRanger = validateAndCategorizeResources(
+ RANGER, strSet("arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2/*", "arn:aws:s3:::bucket3/key.txt"));
+ assertThat(resultRanger).containsOnly(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket2", "*", null),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket3", null, "key.txt"));
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithInvalidArnThrows() {
+ final String invalidArn = "arn:aws:ec2:::bucket";
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton(invalidArn)),
+ "IAM session policy: Unsupported Resource Arn - " + invalidArn, NOT_SUPPORTED_OPERATION);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, Collections.singleton(invalidArn)),
+ "IAM session policy: Unsupported Resource Arn - " + invalidArn, NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithArnWithNoBucketThrows() {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, Collections.singleton("arn:aws:s3:::")),
+ "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, Collections.singleton("arn:aws:s3:::")),
+ "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testValidateAndCategorizeResourcesWithNoResourcesThrows() {
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(NATIVE, emptySet()), "IAM session policy: No Resource(s) found in policy",
+ MALFORMED_POLICY_DOCUMENT);
+ expectOMExceptionWithCode(
+ () -> validateAndCategorizeResources(RANGER, emptySet()), "IAM session policy: No Resource(s) found in policy",
+ MALFORMED_POLICY_DOCUMENT);
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithResourceAny() {
+ // This also tests that acls are deduplicated across different resource types
+ final Set<S3Action> actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET, S3Action.GET_OBJECT)
+ .collect(Collectors.toSet()); // actions at volume, bucket and key levels
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.ANY, "*", null, null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type ANY not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ final Set<IOzoneObj> readAndListObjects = objSet(volume(), bucket("*")); // volume, bucket level have READ, LIST
+ final Set<IOzoneObj> readObject = objSet(key("*", "*")); // key level has READ
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObjects, acls(READ, LIST)),
+ new OzoneGrant(readObject, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketResourceThatIsListBucket() {
+ final Set<S3Action> actions = Collections.singleton(IamSessionPolicyResolver.S3Action.LIST_BUCKET);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> readAndListObject = objSet(bucket("bucket1"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), prefix("bucket1", ""));
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), key("bucket1", "*"));
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObject, acls(READ, LIST)), new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketResourceThatIsNotListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.CREATE_BUCKET);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> createObject = objSet(bucket("bucket1"));
+ final Set<IOzoneObj> readObject = objSet(volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(createObject, acls(CREATE)), new OzoneGrant(readObject, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketWildcardResource() {
+ final Set<S3Action> actions = Collections.singleton(IamSessionPolicyResolver.S3Action.PUT_BUCKET_ACL);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "bucket1*", null, null));
+ final Set<IOzoneObj> readReadAclAndWriteAclObject = objSet(bucket("bucket1*"));
+ final Set<IOzoneObj> readVolume = objSet(volume());
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readReadAclAndWriteAclObject, acls(READ, READ_ACL, WRITE_ACL)),
+ new OzoneGrant(readVolume, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithBucketsWildcardResourceAll() {
+ // For AWS IAM, s3:ListAllMyBuckets supports both "*" and "arn:aws:s3:::*" as
+ // Resource values. The "*" case is covered by testCreatePathsAndPermissionsWithResourceAny.
+ // This test ensures that "arn:aws:s3:::*" (parsed as BUCKET_WILDCARD with bucket="*")
+ // also grants the expected volume-level permissions for ListAllMyBuckets.
+ final Set<S3Action> actions = Stream.of(S3Action.LIST_ALL_MY_BUCKETS, S3Action.LIST_BUCKET)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET_WILDCARD, "*", null, null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type BUCKET_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+
+ // Both the volume and the wildcard bucket should end up with READ + LIST permissions.
+ // We also need READ access on the keys
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ final Set<IOzoneObj> readAndListObjects = objSet(volume(), bucket("*"));
+ final Set<IOzoneObj> readObjects = objSet(key("*", "*"));
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndListObjects, acls(READ, LIST)), new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectExactResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readObjects = objSet(key("bucket1", "key.txt"), bucket("bucket1"), volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(readObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithDeleteObjectGrantsDeleteOnKey() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.DELETE_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readVolumeAndBucket = objSet(volume(), bucket("bucket1"));
+ final Set<IOzoneObj> deleteKey = objSet(key("bucket1", "key.txt"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(deleteKey, acls(DELETE)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithAbortMultipartUploadGrantsWriteOnKey() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.ABORT_MULTIPART_UPLOAD);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readVolumeAndBucket = objSet(volume(), bucket("bucket1"));
+ final Set<IOzoneObj> writeKey = objSet(key("bucket1", "key.txt"));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readVolumeAndBucket, acls(READ)), new OzoneGrant(writeKey, acls(WRITE)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectPrefixResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "prefix/", null));
+ final Set<IOzoneObj> nativeReadObjects = objSet(prefix("bucket1", "prefix/"), bucket("bucket1"), volume());
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type OBJECT_PREFIX not supported for RangerOzoneAuthorizer");
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithObjectPrefixWildcardResource() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "prefix/*", null));
+
+ expectIllegalArgumentException(
+ () -> createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, new LinkedHashMap<>()),
+ "ResourceSpec type OBJECT_PREFIX_WILDCARD not supported for OzoneNativeAuthorizer");
+
+ final Set<IOzoneObj> rangerReadObjects = objSet(key("bucket1", "prefix/*"), bucket("bucket1"), volume());
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForObjectActionMustIgnoreConditionPrefixes() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", "", null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ final Set<IOzoneObj> nativeReadObjects = objSet(prefix("bucket1", ""), bucket("bucket1"), volume());
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactly(new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", "*", null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ final Set<IOzoneObj> rangerReadObjects = objSet(key("bucket1", "*"), bucket("bucket1"), volume());
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactly(new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.LIST_BUCKET);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> nativeReadObjects = objSet(
+ prefix("bucket1", "folder1/"), prefix("bucket1", "folder2/"), volume());
+ final Set<IOzoneObj> nativeReadAndListObject = objSet(bucket("bucket1"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(nativeReadObjects, acls(READ)), new OzoneGrant(nativeReadAndListObject, acls(READ, LIST)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Set<IOzoneObj> rangerReadObjects = objSet(
+ key("bucket1", "folder1/"), key("bucket1", "folder2/"), volume());
+ final Set<IOzoneObj> rangerReadAndListObject = objSet(bucket("bucket1"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(rangerReadObjects, acls(READ)), new OzoneGrant(rangerReadAndListObject, acls(READ, LIST)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithConditionPrefixesForBucketActionWhenActionIsNotListBucket() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_BUCKET_ACL);
+ final Set<String> prefixes = strSet("folder1/", "folder2/");
+ final IamSessionPolicyResolver.Condition condition = new IamSessionPolicyResolver.Condition(
+ "StringEquals", prefixes);
+ final Set<IOzoneObj> readObject = objSet(volume());
+ final Set<IOzoneObj> readAndReadAclObject = objSet(bucket("bucket1"));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, condition, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL)));
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, condition, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readObject, acls(READ)), new OzoneGrant(readAndReadAclObject, acls(READ, READ_ACL)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithNoMappedActions() {
+ final Set<S3Action> actions = emptySet();
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> nativeResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, nativeResourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).isEmpty();
+
+ final Set<IamSessionPolicyResolver.ResourceSpec> rangerResourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_PREFIX_WILDCARD, "bucket1", null, null));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, rangerResourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).isEmpty();
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithNoMappedResources() {
+ final Set<S3Action> actions = Collections.singleton(S3Action.GET_OBJECT);
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = emptySet();
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).isEmpty();
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).isEmpty();
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsDeduplicatesAcrossSameResourceTypes() {
+ final Set<S3Action> actions = Stream.of(
+ S3Action.GET_OBJECT, S3Action.GET_OBJECT_TAGGING, S3Action.DELETE_OBJECT, S3Action.DELETE_OBJECT_TAGGING)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Collections.singleton(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"));
+ final Set<IOzoneObj> readAndDeleteAndWriteObject = objSet(key("bucket1", "key.txt"));
+ final Set<IOzoneObj> readObjects = objSet(bucket("bucket1"), volume());
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)),
+ new OzoneGrant(readObjects, acls(READ)));
+
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(readAndDeleteAndWriteObject, acls(READ, DELETE, WRITE)),
+ new OzoneGrant(readObjects, acls(READ)));
+ }
+
+ @Test
+ public void testCreatePathsAndPermissionsWithAllS3ActionsOverridesAnyOtherAction() {
+ final Set<S3Action> actions = Stream.of(
+ S3Action.ALL_S3, S3Action.GET_OBJECT, S3Action.DELETE_OBJECT, S3Action.LIST_BUCKET)
+ .collect(Collectors.toSet());
+ final Set<IamSessionPolicyResolver.ResourceSpec> resourceSpecs = Stream.of(
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.OBJECT_EXACT, "bucket1", null, "key.txt"),
+ new IamSessionPolicyResolver.ResourceSpec(S3ResourceType.BUCKET, "bucket2", null, null))
+ .collect(Collectors.toSet());
+ final Set<IOzoneObj> allObjects = objSet(key("bucket1", "key.txt"), bucket("bucket2"));
+
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), bucket("bucket1"), prefix("bucket2", ""));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapNative = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, NATIVE, actions, resourceSpecs, null, objToAclsMapNative);
+ final Set<OzoneGrant> resultNative = groupObjectsByAcls(objToAclsMapNative);
+ assertThat(resultNative).containsExactlyInAnyOrder(
+ new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(nativeReadObjects, acls(READ)));
+
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), bucket("bucket1"), key("bucket2", "*"));
+ final Map<IOzoneObj, Set<ACLType>> objToAclsMapRanger = new LinkedHashMap<>();
+ createPathsAndPermissions(VOLUME, RANGER, actions, resourceSpecs, null, objToAclsMapRanger);
+ final Set<OzoneGrant> resultRanger = groupObjectsByAcls(objToAclsMapRanger);
+ assertThat(resultRanger).containsExactlyInAnyOrder(
+ new OzoneGrant(allObjects, acls(ALL)), new OzoneGrant(rangerReadObjects, acls(READ)));
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsWhenSameStatementsArePresent() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsForSameActionsButDifferentResource() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket2\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"), bucket("my-bucket2"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(
+ objSet(volume(), prefix("my-bucket2", ""), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(
+ objSet(volume(), key("my-bucket2", "*"), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsForDifferentActionsButSameResource() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:CreateBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL, CREATE);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL, CREATE; volume, key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testDeduplicatesAcrossMultipleStatementsWhenAllActionPresent() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket ALL (instead of individual actions); volume and prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket ALL (instead of individual actions); volume and key "*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllowGetPutOnKey() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\"s3:GetObject\", \"s3:PutObject\"],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/folder/file.txt\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedFromBothAuthorizers = new LinkedHashSet<>();
+ // Expected: READ, CREATE, WRITE on key; bucket READ; volume READ
+ final Set<IOzoneObj> keySet = objSet(key("my-bucket", "folder/file.txt"));
+ final Set<ACLType> keyAcls = acls(READ, CREATE, WRITE);
+ expectedResolvedFromBothAuthorizers.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ expectedResolvedFromBothAuthorizers.add(new OzoneGrant(keySet, keyAcls));
+
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers);
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedFromBothAuthorizers);
+ }
+
+ @Test
+ public void testAllActionsForKey() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all key ACLs on prefix "" under bucket; bucket READ, volume READ
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> allKeyAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, allKeyAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: all key acls for resource type KEY with key name "*"
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, allKeyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsForKeyWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected - only ListBucket supports s3:prefix and that is a bucket action,
+ // not object action
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testAllActionsForBucket() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all Bucket ACLs for bucket; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> allBucketAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("my-bucket", "")), acls(READ)));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, allBucketAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: all Bucket ACLs for bucket; volume, key "*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("my-bucket", "*")), acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, allBucketAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsForBucketWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ;
+ // prefix "team/folder", "team/folder/" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(
+ new OzoneGrant(objSet(volume(), prefix("my-bucket", "team/folder"), prefix("my-bucket", "team/folder/")),
+ acls(READ)));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: READ, LIST ACLs for bucket (only ListBucket supports s3:prefix); volume READ,
+ // key "team/folder", "team/folder/*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(
+ new OzoneGrant(objSet(volume(), key("my-bucket", "team/folder"), key("my-bucket", "team/folder/*")),
+ acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesInSeparateStatements() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetBucketAcl\",\n" +
+ " \"s3:PutBucketAcl\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::my-bucket/*\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ // Expected for native: all key ACLs on prefix "" under bucket
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> keyAllAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, LIST, READ_ACL, WRITE_ACL; volume READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ // Expected for Ranger: all key acls for resource type KEY with key name "*"
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesInOneStatement() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:*\"\n" +
+ " ],\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all for bucket and key acls; volume READ
+ final Set<IOzoneObj> resourceSetNative = objSet(bucket("my-bucket"), prefix("my-bucket", ""));
+ expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(ALL)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: all for bucket and key acls; volume READ
+ final Set<IOzoneObj> resourceSetRanger = objSet(bucket("my-bucket"), key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(ALL)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultipleResourcesWithDifferentBucketsAndDeepPathsInOneStatement() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:*\"\n" +
+ " ],\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket/team/folder1/security/*\",\n" +
+ " \"arn:aws:s3:::my-bucket2/team/folder2/misc/*\"\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: all key ACLs on prefix "team/folder1/security/" under
+ // my-bucket and all key ACLs on prefix "team/folder2/misc/" under my-bucket2; bucket READ; volume READ
+ final Set<IOzoneObj> keyPrefixSet = objSet(
+ prefix("my-bucket", "team/folder1/security/"), prefix("my-bucket2", "team/folder2/misc/"));
+ final Set<ACLType> keyAllAcls = acls(ALL);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAllAcls));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: all key acls for resource type KEY with key name
+ // "team/folder1/security/*" under my-bucket and "team/folder2/misc/*" under my-bucket2; bucket READ; volume READ
+ final Set<IOzoneObj> rangerKeySet = objSet(
+ key("my-bucket", "team/folder1/security/*"), key("my-bucket2", "team/folder2/misc/*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAllAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("my-bucket"), bucket("my-bucket2")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testUnsupportedActionIgnoredWhenItIsTheOnlyAction() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ReplicateObject\",\n" + // unsupported action
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testUnsupportedResourceArnThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:dynamodb:us-east-2:123456789012:table/example-table\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Unsupported Resource Arn - " +
+ "arn:aws:dynamodb:us-east-2:123456789012:table/example-table", NOT_SUPPORTED_OPERATION);
+ }
+
+ @Test
+ public void testListBucketWithWildcard() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::proj-*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ and LIST on wildcard pattern; volume and key "*" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("proj-*"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("proj-*", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testListBucketOperationsWithNoPrefixes() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:ListBucket\",\n" +
+ " \"s3:ListBucketMultipartUploads\"\n" +
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::proj\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ and LIST; volume, prefix "" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("proj"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ final Set<IOzoneObj> nativeReadObjects = objSet(volume(), prefix("proj", ""));
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(nativeReadObjects, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger: bucket READ and LIST; volume, key "*" READ
+ final Set<IOzoneObj> rangerReadObjects = objSet(volume(), key("proj", "*"));
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(rangerReadObjects, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testIgnoresUnsupportedActionsWhenSupportedActionsAreIncluded() throws OMException {
+ final String json = "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Sid\": \"AllowListingOfDataLakeFolder\",\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:GetAccelerateConfiguration\",\n" + // unsupported action
+ " \"s3:GetBucketAcl\",\n" + // ignored because it doesn't support s3:prefix condition
+ " \"s3:GetObject\",\n" + // object-level action not applied for bucket
+ " \"s3:GetObjectAcl\",\n" + // unsupported action
+ " \"s3:ListBucket\",\n" +
+ " \"s3:ListBucketMultipartUploads\"\n" + // ignored because it doesn't support s3:prefix condition
+ " ],\n" +
+ " \"Resource\": \"arn:aws:s3:::bucket1\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+
+ // Expected for native: READ, LIST bucket acls; volume and prefixes "team/folder", "team/folder/" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("bucket1"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedNative.add(new OzoneGrant(
+ objSet(volume(), prefix("bucket1", "team/folder"), prefix("bucket1", "team/folder/")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST bucket acls; volume and keys "team/folder" and "team/folder/*" READ
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(
+ objSet(volume(), key("bucket1", "team/folder"), key("bucket1", "team/folder/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMultiplePrefixesWithWildcards() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/*\",\n" +
+ " \"Condition\": { \"StringEquals\": { \"s3:prefix\": [\"a/*\", \"b/*\"] } }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // s3:prefix conditions do not apply to object actions like s3:GetObject.
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testListAndGetWithPrefixConditionSkipsObjectAction() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\"s3:ListBucket\", \"s3:GetObject\"],\n" +
+ " \"Resource\": [\"arn:aws:s3:::logs\", \"arn:aws:s3:::logs/*\"],\n" +
+ " \"Condition\": { \"StringLike\": { \"s3:prefix\": \"team/*\" } }\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Expected for native (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
+ // prefix "log/team" READ
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ expectedResolvedNative.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume(), prefix("logs", "team/")), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ // Expected for Ranger (GetObject is ignored because s3:prefix is present): READ, LIST bucket acls; volume READ;
+ // key "log/team/*" READ
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("logs")), acls(READ, LIST)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), key("logs", "team/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testListBucketOnObjectResourceReturnsEmpty() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testObjectResourceWithWildcardInMiddle() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::logs/file*.log\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards in middle of object resource are not supported for Native authorizer
+ expectResolveThrows(
+ json, NATIVE, "IAM session policy: Wildcard prefix patterns are not supported for Ozone native " +
+ "authorizer if wildcard is not at the end", NOT_SUPPORTED_OPERATION);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ acl on key "file*.log", bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("logs", "file*.log"), bucket("logs"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testObjectResourceWithPrefixWildcard() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" +
+ " \"Resource\": \"arn:aws:s3:::myBucket/file*\"\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ acl on prefix "file" under bucket, bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("myBucket", "file"), bucket("myBucket"), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ acl on key "file*", bucket READ, volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("myBucket", "file*"), bucket("myBucket"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testBucketActionOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"s3:ListAllMyBuckets\",\n" +
+ " \"s3:ListBucket\"\n" +
+ " ],\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ and LIST on volume and bucket (wildcard), READ on key "*"
+ final Set<IOzoneObj> resourceSet = objSet(volume(), bucket("*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(READ, LIST)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testObjectActionOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:PutObject\",\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: CREATE and WRITE key acls on wildcard pattern, bucket READ, volume READ
+ final Set<IOzoneObj> keySet = objSet(key("*", "*"));
+ final Set<ACLType> keyAcls = acls(CREATE, WRITE);
+ expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllResourcesWithPrefixCondition() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"*\",\n" +
+ " \"Condition\": {\n" +
+ " \"StringLike\": {\n" +
+ " \"s3:prefix\": [ \"team/folder\", \"team/folder/*\" ]\n" +
+ " }\n" +
+ " }\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: (only ListBucket supports s3:prefix) READ volume; READ, LIST acl on bucket;
+ // READ on key "team/folder", "team/folder/*"
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("*")), acls(READ, LIST)));
+ expectedResolvedRanger.add(
+ new OzoneGrant(objSet(volume(), key("*", "team/folder"), key("*", "team/folder/*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST acl on volume, ALL acl bucket (wildcard) and key (wildcard)
+ final Set<IOzoneObj> resourceSet = objSet(bucket("*"), key("*", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSet, acls(ALL)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllBucketResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: ALL bucket acls on wildcard pattern, volume READ, key "*" READ
+ final Set<IOzoneObj> bucketSet = objSet(bucket("*"));
+ final Set<ACLType> bucketAcls = acls(ALL);
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(key("*", "*")), acls(READ)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ, LIST)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testAllActionsOnAllObjectResources() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:*\",\n" +
+ " \"Resource\": \"arn:aws:s3:::*/*\"\n" +
+ " }]\n" +
+ "}";
+
+ // Wildcards on bucket are not supported for Native authorizer
+ expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(json);
+
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: ALL key acls on wildcard pattern; bucket READ; volume READ
+ final Set<IOzoneObj> keySet = objSet(key("*", "*"));
+ final Set<ACLType> keyAcls = acls(ALL);
+ expectedResolvedRanger.add(new OzoneGrant(keySet, keyAcls));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume(), bucket("*")), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupGetStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Get*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, READ_ACL acls
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, READ_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for native: READ acl on prefix "" under bucket; volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("my-bucket", ""), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, READ_ACL acls
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("my-bucket", "*"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupListStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:List*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" + // ListMultipartUploadParts has READ effect on file/object resources
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: READ, LIST bucket acls
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcls = acls(READ, LIST);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for native: READ acl on prefix "" under bucket; volume READ
+ final Set<IOzoneObj> readObjectsNative = objSet(prefix("my-bucket", ""), volume());
+ expectedResolvedNative.add(new OzoneGrant(readObjectsNative, acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: READ, LIST bucket acls
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcls));
+ // Expected for Ranger: READ key acl for resource type KEY with key name "*"; volume READ
+ final Set<IOzoneObj> readObjectsRanger = objSet(key("my-bucket", "*"), volume());
+ expectedResolvedRanger.add(new OzoneGrant(readObjectsRanger, acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupPutStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Put*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: bucket READ, READ_ACL, WRITE_ACL acl
+ final Set<IOzoneObj> bucketSet = objSet(bucket("my-bucket"));
+ final Set<ACLType> bucketAcl = acls(READ, READ_ACL, WRITE_ACL);
+ expectedResolvedNative.add(new OzoneGrant(bucketSet, bucketAcl));
+ // Expected for native: CREATE, WRITE acls on prefix "" under bucket
+ final Set<IOzoneObj> keyPrefixSet = objSet(prefix("my-bucket", ""));
+ final Set<ACLType> keyAcls = acls(CREATE, WRITE);
+ expectedResolvedNative.add(new OzoneGrant(keyPrefixSet, keyAcls));
+ // Expected for native: volume READ
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: bucket READ, READ_ACL, WRITE_ACL acl
+ expectedResolvedRanger.add(new OzoneGrant(bucketSet, bucketAcl));
+ // Expected for Ranger: CREATE, WRITE key acls for resource type KEY with key name "*"
+ final Set<IOzoneObj> rangerKeySet = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(rangerKeySet, keyAcls));
+ // Expected for Ranger: volume READ
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testWildcardActionGroupDeleteStar() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:Delete*\",\n" +
+ " \"Resource\": [\n" +
+ " \"arn:aws:s3:::my-bucket\",\n" +
+ " \"arn:aws:s3:::my-bucket/*\"\n" +
+ " ]\n" +
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ final Set<OzoneGrant> expectedResolvedNative = new LinkedHashSet<>();
+ // Expected for native: DELETE and WRITE on prefix "" under bucket; bucket READ, DELETE; volume READ
+ final Set<IOzoneObj> resourceSetNative = objSet(prefix("my-bucket", ""));
+ expectedResolvedNative.add(new OzoneGrant(resourceSetNative, acls(DELETE, WRITE)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE)));
+ expectedResolvedNative.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromNativeAuthorizer).isEqualTo(expectedResolvedNative);
+
+ final Set<OzoneGrant> expectedResolvedRanger = new LinkedHashSet<>();
+ // Expected for Ranger: DELETE and WRITE on resource type KEY with key name "*"; bucket READ, DELETE; volume READ
+ final Set<IOzoneObj> resourceSetRanger = objSet(key("my-bucket", "*"));
+ expectedResolvedRanger.add(new OzoneGrant(resourceSetRanger, acls(DELETE, WRITE)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(bucket("my-bucket")), acls(READ, DELETE)));
+ expectedResolvedRanger.add(new OzoneGrant(objSet(volume()), acls(READ)));
+ assertThat(resolvedFromRangerAuthorizer).isEqualTo(expectedResolvedRanger);
+ }
+
+ @Test
+ public void testMismatchedActionAndResourceReturnsEmpty() throws OMException {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:GetObject\",\n" + // object-level action
+ " \"Resource\": \"arn:aws:s3:::my-bucket\"\n" + // bucket-level resource
+ " }]\n" +
+ "}";
+
+ final Set<OzoneGrant> resolvedFromNativeAuthorizer = resolve(json, VOLUME, NATIVE);
+ final Set<OzoneGrant> resolvedFromRangerAuthorizer = resolve(json, VOLUME, RANGER);
+
+ // Ensure what we got is what we expected
+ assertThat(resolvedFromNativeAuthorizer).isEmpty();
+ assertThat(resolvedFromRangerAuthorizer).isEmpty();
+ }
+
+ @Test
+ public void testInvalidResourceArnThrows() {
+ final String json = "{\n" +
+ " \"Statement\": [{\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": \"s3:ListBucket\",\n" +
+ " \"Resource\": \"arn:aws:s3:::\"\n" +
+ " }]\n" +
+ "}";
+
+ expectResolveThrowsForBothAuthorizers(
+ json, "IAM session policy: Invalid Resource Arn - arn:aws:s3:::", MALFORMED_POLICY_DOCUMENT);
+ }
+
+ private static void expectIllegalArgumentException(Runnable runnable, String expectedMessage) {
+ try {
+ runnable.run();
+ throw new AssertionError("Expected exception not thrown");
+ } catch (IllegalArgumentException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ }
+ }
+
+ private static void expectOMExceptionWithCode(RunnableThrowingOMException runnable, String expectedMessage,
+ OMException.ResultCodes expectedCode) {
+ try {
+ runnable.run();
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ assertThat(ex.getResult()).isEqualTo(expectedCode);
+ }
+ }
+
+ @FunctionalInterface
+ private interface RunnableThrowingOMException {
+ void run() throws OMException;
+ }
+
+ private static IOzoneObj key(String bucket, String key) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.KEY)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .setKeyName(key)
+ .build();
+ }
+
+ private static IOzoneObj volume() {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.VOLUME)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .build();
+ }
+
+ private static IOzoneObj bucket(String bucket) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.BUCKET)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .build();
+ }
+
+ private static IOzoneObj prefix(String bucket, String prefix) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.PREFIX)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(VOLUME)
+ .setBucketName(bucket)
+ .setPrefixName(prefix)
+ .build();
+ }
+
+ private static Set<IOzoneObj> objSet(IOzoneObj... objs) {
+ final Set<IOzoneObj> s = new LinkedHashSet<>();
+ Collections.addAll(s, objs);
+ return s;
+ }
+
+ private static Set<ACLType> acls(ACLType... types) {
+ final Set<ACLType> s = new LinkedHashSet<>();
+ Collections.addAll(s, types);
+ return s;
+ }
+
+ private static Set<String> strSet(String... strs) {
+ final Set<String> s = new LinkedHashSet<>();
+ Collections.addAll(s, strs);
+ return s;
+ }
+
+ private static void expectResolveThrows(String json, IamSessionPolicyResolver.AuthorizerType authorizerType,
+ String expectedMessage, OMException.ResultCodes expectedCode) {
+ try {
+ resolve(json, VOLUME, authorizerType);
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(expectedMessage);
+ assertThat(ex.getResult()).isEqualTo(expectedCode);
+ }
+ }
+
+ private static void expectResolveThrowsForBothAuthorizers(String json, String expectedMessage,
+ OMException.ResultCodes expectedCode) {
+ expectResolveThrows(json, NATIVE, expectedMessage, expectedCode);
+ expectResolveThrows(json, RANGER, expectedMessage, expectedCode);
+ }
+
+ /**
+ * Ensure resources containing wildcards in buckets throw an Exception
+ * when the OzoneNativeAuthorizer is used.
+ */
+ private static void expectBucketWildcardUnsupportedExceptionForNativeAuthorizer(String json) {
+ try {
+ resolve(json, VOLUME, NATIVE);
+ throw new AssertionError("Expected exception not thrown");
+ } catch (OMException ex) {
+ assertThat(ex.getMessage()).isEqualTo(
+ "IAM session policy: Wildcard bucket patterns are not supported for Ozone native authorizer");
+ assertThat(ex.getResult()).isEqualTo(NOT_SUPPORTED_OPERATION);
+ }
+ }
+
+ private static String createJsonStringLargerThan2048Characters() {
+ final StringBuilder jsonBuilder = new StringBuilder();
- jsonBuilder.append("{\n");
- jsonBuilder.append(" \"Statement\": [{\n");
- jsonBuilder.append(" \"Effect\": \"Allow\",\n");
- jsonBuilder.append(" \"Action\": \"s3:ListBucket\",\n");
- jsonBuilder.append(" \"Resource\": \"arn:aws:s3:::");
++ jsonBuilder.append("{\n")
++ .append(" \"Statement\": [{\n")
++ .append(" \"Effect\": \"Allow\",\n")
++ .append(" \"Action\": \"s3:ListBucket\",\n")
++ .append(" \"Resource\": \"arn:aws:s3:::");
+ // Add enough characters to exceed 2048
+ while (jsonBuilder.length() < 2048) {
+ jsonBuilder.append("very-long-bucket-name-that-exceeds-the-limit-");
+ }
- jsonBuilder.append("\"\n");
- jsonBuilder.append(" }]\n");
- jsonBuilder.append('}');
++ jsonBuilder.append("\"\n")
++ .append(" }]\n")
++ .append('}');
+ return jsonBuilder.toString();
+ }
+
+ private static String create2048CharJsonString() {
+ final StringBuilder jsonBuilder = new StringBuilder();
- jsonBuilder.append("{\n");
- jsonBuilder.append(" \"Statement\": [{\n");
- jsonBuilder.append(" \"Effect\": \"Allow\",\n");
- jsonBuilder.append(" \"Action\": \"s3:ListBucket\",\n");
- jsonBuilder.append(" \"Resource\": \"arn:aws:s3:::");
++ jsonBuilder.append("{\n")
++ .append(" \"Statement\": [{\n")
++ .append(" \"Effect\": \"Allow\",\n")
++ .append(" \"Action\": \"s3:ListBucket\",\n")
++ .append(" \"Resource\": \"arn:aws:s3:::");
+ // Add characters to reach exactly 2048 (accounting for closing brackets and newlines)
+ // Closing part: "\"\n }]\n}" = 8 chars
+ while (jsonBuilder.length() < 2048 - 8) {
+ jsonBuilder.append('a');
+ }
+ jsonBuilder.append("\"\n }]\n}");
+ return jsonBuilder.toString();
+ }
+}
+
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index edb1484fd0f,d4a340b311a..f9078360799
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -46,10 -43,6 +44,7 @@@ public class TestOzoneConfigurationFiel
ReconConfigKeys.class, ReconServerConfigKeys.class,
S3GatewayConfigKeys.class,
S3SecretConfigKeys.class,
- S3STSConfigKeys.class,
- SCMHTTPServerConfig.class,
- SCMHTTPServerConfig.ConfigStrings.class,
- ScmConfig.ConfigStrings.class
++ S3STSConfigKeys.class
};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 9bb0d801ee7,22c71cc2985..dde32437db5
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -156,9 -156,7 +156,10 @@@ enum Type
PutObjectTagging = 140;
GetObjectTagging = 141;
DeleteObjectTagging = 142;
- AssumeRole = 143;
- RevokeSTSToken = 144;
- DeleteRevokedSTSTokens = 145;
+ SubmitSnapshotDiff = 143;
++ AssumeRole = 144;
++ RevokeSTSToken = 145;
++ DeleteRevokedSTSTokens = 146;
}
enum SafeMode {
@@@ -309,10 -307,8 +310,11 @@@ message OMRequest
optional PutObjectTaggingRequest putObjectTaggingRequest = 141;
optional DeleteObjectTaggingRequest deleteObjectTaggingRequest = 142;
repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 143;
- optional AssumeRoleRequest assumeRoleRequest = 144;
- optional RevokeSTSTokenRequest revokeSTSTokenRequest = 145;
- optional DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = 146;
- optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 147;
-
- optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
++ optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
++ optional AssumeRoleRequest assumeRoleRequest = 145;
++ optional RevokeSTSTokenRequest revokeSTSTokenRequest = 146;
++ optional DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = 147;
++ optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 148;
}
message OMResponse {
@@@ -443,12 -439,11 +445,13 @@@
optional GetQuotaRepairStatusResponse GetQuotaRepairStatusResponse = 136;
optional StartQuotaRepairResponse StartQuotaRepairResponse = 137;
- optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
- optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
- optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
-
- optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
+ optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
+ optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
+ optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
- optional AssumeRoleResponse assumeRoleResponse = 143;
- optional RevokeSTSTokenResponse revokeSTSTokenResponse = 144;
- optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 145;
++ optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
++ optional AssumeRoleResponse assumeRoleResponse = 144;
++ optional RevokeSTSTokenResponse revokeSTSTokenResponse = 145;
++ optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 146;
}
enum Status {
@@@ -579,8 -574,11 +582,13 @@@
TOO_MANY_SNAPSHOTS = 98;
- REVOKED_TOKEN = 99;
- MALFORMED_POLICY_DOCUMENT = 100;
+ ETAG_MISMATCH = 99;
+
+ ETAG_NOT_AVAILABLE = 100;
+
+ ATOMIC_WRITE_CONFLICT = 101;
++ REVOKED_TOKEN = 102;
++ MALFORMED_POLICY_DOCUMENT = 103;
}
/**
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index b14f01cf6ba,a5ba074156e..46573917c88
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@@ -645,26 -596,12 +645,26 @@@ public class OmMetadataReader implement
"Bucket:" + obj.getBucketName() + " " : "";
String keyName = obj.getKeyName() != null ?
"Key:" + obj.getKeyName() : "";
+ // For STS tokens, make clear that the user is using an assumed role, otherwise the access denied
+ // message could be confusing
+ String user = normalizedRequestContext.getClientUgi().getShortUserName();
+ final STSTokenIdentifier stsTokenIdentifier = OzoneManager.getStsTokenIdentifier();
+ if (stsTokenIdentifier != null) {
- final StringBuilder builder = new StringBuilder(user);
- builder.append(" (STS assumed role arn = ");
- builder.append(stsTokenIdentifier.getRoleArn());
- builder.append(", tempAccessKeyId = ");
- builder.append(stsTokenIdentifier.getTempAccessKeyId());
- builder.append(')');
++ final StringBuilder builder = new StringBuilder(user)
++ .append(" (STS assumed role arn = ")
++ .append(stsTokenIdentifier.getRoleArn())
++ .append(", tempAccessKeyId = ")
++ .append(stsTokenIdentifier.getTempAccessKeyId())
++ .append(')');
+ user = builder.toString();
+ }
log.warn("User {} doesn't have {} permission to access {} {}{}{}",
- context.getClientUgi().getShortUserName(), context.getAclRights(),
+ user,
+ normalizedRequestContext.getAclRights(),
obj.getResourceType(), volumeName, bucketName, keyName);
throw new OMException(
- "User " + context.getClientUgi().getShortUserName() +
- " doesn't have " + context.getAclRights() +
+ "User " + user +
+ " doesn't have " + normalizedRequestContext.getAclRights() +
" permission to access " + obj.getResourceType() + " " +
volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d1f27c6132c,d258324b3bb..1986b34acb3
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -684,12 -695,7 +704,13 @@@ public final class OzoneManager extend
this.isS3MultiTenancyEnabled =
OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf);
+ // Enable S3 STS if config key is set
+ this.isS3STSEnabled = conf.getBoolean(
+ OzoneConfigKeys.OZONE_S3G_STS_HTTP_ENABLED_KEY,
+ OzoneConfigKeys.OZONE_S3G_STS_HTTP_ENABLED_DEFAULT);
+
- metrics = OMMetrics.create();
+ metrics = OMMetrics.create(conf);
++
omSnapshotIntMetrics = OmSnapshotInternalMetrics.create();
perfMetrics = OMPerformanceMetrics.register();
omDeletionMetrics = DeletingServiceMetrics.create();
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
index 1ba4b7186f2,00000000000..8c13aac5190
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
@@@ -1,285 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.UUID;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.security.token.ShortLivedTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+
+/**
+ * Token identifier for STS (Security Token Service) tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class STSTokenIdentifier extends ShortLivedTokenIdentifier {
+ public static final Text KIND_NAME = new Text("STSToken");
+
+ // STS-specific fields
+ private String roleArn;
+ private String originalAccessKeyId;
+ private String secretAccessKey;
+ private String sessionPolicy;
+
+ // Encryption key derived from ManagedSecretKey for this token
+ private transient byte[] encryptionKey;
+
+ // Service name for STS tokens
+ public static final String STS_SERVICE = "STS";
+
+ /**
+ * Create an empty STS token identifier.
+ */
+ public STSTokenIdentifier() {
+ super();
+ }
+
+ /**
+ * Create a new STS token identifier with encryption support.
+ *
+ * @param tempAccessKeyId the temporary access key ID (owner)
+ * @param originalAccessKeyId the original long-lived access key ID that created this token
+ * @param roleArn the ARN of the assumed role
+ * @param expiry the token expiration time
+ * @param secretAccessKey the secret access key associated with the temporary access key ID
+ * @param sessionPolicy an optional opaque identifier that further limits the scope of
+ * the permissions granted by the role
+ * @param encryptionKey the key bytes for encrypting sensitive fields
+ */
+ public STSTokenIdentifier(String tempAccessKeyId, String originalAccessKeyId, String roleArn, Instant expiry,
+ String secretAccessKey, String sessionPolicy, byte[] encryptionKey) {
+ super(tempAccessKeyId, expiry);
+ this.originalAccessKeyId = originalAccessKeyId;
+ this.roleArn = roleArn;
+ this.secretAccessKey = secretAccessKey;
+ this.sessionPolicy = sessionPolicy;
+ this.encryptionKey = encryptionKey != null ? encryptionKey.clone() : null;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String getService() {
+ return STS_SERVICE;
+ }
+
+ @Override
+ public void readFromByteArray(byte[] bytes) throws IOException {
+ final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
+ readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.write(toProtoBuf().toByteArray());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ fromProtoBuf(token);
+ }
+
+ /**
+ * Convert this identifier to protobuf format.
+ */
+ public OMTokenProto toProtoBuf() {
+ Preconditions.checkArgument(this.encryptionKey != null, "The encryption key must not be null");
+
+ final OMTokenProto.Builder builder = OMTokenProto.newBuilder();
+ // Note: secretKeyId must be set before attempting to decrypt secretAccessKey
+ if (getSecretKeyId() != null) {
+ builder.setSecretKeyId(getSecretKeyId().toString());
+ }
+
+ builder
+ .setType(OMTokenProto.Type.S3_STS_TOKEN)
+ .setMaxDate(getExpiry().toEpochMilli())
+ .setOwner(getOwnerId() != null ? getOwnerId() : "")
+ .setAccessKeyId(getOwnerId() != null ? getOwnerId() : "")
+ .setOriginalAccessKeyId(originalAccessKeyId != null ? originalAccessKeyId : "")
+ .setRoleArn(roleArn != null ? roleArn : "")
+ .setSecretAccessKey(secretAccessKey != null ? encryptSensitiveField(secretAccessKey) : "")
+ .setSessionPolicy(sessionPolicy != null ? sessionPolicy : "");
+
+ return builder.build();
+ }
+
+ /**
+ * Initialize this identifier from protobuf.
+ */
+ public void fromProtoBuf(OMTokenProto token) throws IOException {
+ Preconditions.checkArgument(
+ token.getType() == OMTokenProto.Type.S3_STS_TOKEN,
+ "Invalid token type for STSTokenIdentifier: " + token.getType());
+ Preconditions.checkArgument(this.encryptionKey != null, "The encryption key must not be null");
+
+ setOwnerId(token.getOwner());
+ setExpiry(Instant.ofEpochMilli(token.getMaxDate()));
+
+ if (token.hasOriginalAccessKeyId()) {
+ this.originalAccessKeyId = token.getOriginalAccessKeyId();
+ }
+ if (token.hasRoleArn()) {
+ this.roleArn = token.getRoleArn();
+ }
+ if (token.hasSecretKeyId()) {
+ try {
+ setSecretKeyId(UUID.fromString(token.getSecretKeyId()));
+ } catch (IllegalArgumentException e) {
+ // Handle invalid UUID format gracefully
+ throw new IOException(
+ "Invalid secretKeyId format in STS token: " + token.getSecretKeyId(), e);
+ }
+ }
+ // Note: secretKeyId must be set before attempting to decrypt secretAccessKey
+ if (token.hasSecretAccessKey()) {
+ this.secretAccessKey = decryptSensitiveField(token.getSecretAccessKey());
+ }
+
+ if (token.hasSessionPolicy()) {
+ this.sessionPolicy = token.getSessionPolicy();
+ }
+ }
+
+ /**
+ * Encrypt a sensitive field using the configured encryption key.
+ */
+ private String encryptSensitiveField(String value) {
+ if (encryptionKey == null) {
+ throw new IllegalStateException("Encryption key must be set before encrypting sensitive fields");
+ }
+
+ try {
+ final byte[] aad = computeAadBytes();
+ return STSTokenEncryption.encrypt(value, encryptionKey, aad);
+ } catch (STSTokenEncryption.STSTokenEncryptionException e) {
+ throw new RuntimeException("Token encryption failed", e);
+ }
+ }
+
+ /**
+ * Decrypt a sensitive field using the configured encryption key.
+ */
+ private String decryptSensitiveField(String encryptedValue) {
+ if (encryptionKey == null) {
+ throw new IllegalStateException("Encryption key must be set before decrypting sensitive fields");
+ }
+
+ try {
+ final byte[] aad = computeAadBytes();
+ return STSTokenEncryption.decrypt(encryptedValue, encryptionKey, aad);
+ } catch (STSTokenEncryption.STSTokenEncryptionException e) {
+ throw new RuntimeException("Token decryption failed", e);
+ }
+ }
+
+ /**
+ * Compute additional authenticated data to bind token context to encryption.
+ * Includes token type, ownerId, expiry millis, and secretKeyId.
+ */
+ private byte[] computeAadBytes() {
- final StringBuilder stringBuilder = new StringBuilder("v1|S3_STS_TOKEN|");
- stringBuilder.append(getOwnerId());
- stringBuilder.append('|');
- stringBuilder.append(getExpiry().toEpochMilli());
- stringBuilder.append('|');
- stringBuilder.append(getSecretKeyId().toString());
++ final StringBuilder stringBuilder = new StringBuilder("v1|S3_STS_TOKEN|")
++ .append(getOwnerId())
++ .append('|')
++ .append(getExpiry().toEpochMilli())
++ .append('|')
++ .append(getSecretKeyId().toString());
+ final String aad = stringBuilder.toString();
+ return aad.getBytes(StandardCharsets.UTF_8);
+ }
+
+ public String getRoleArn() {
+ return roleArn;
+ }
+
+ public String getSecretAccessKey() {
+ return secretAccessKey;
+ }
+
+ public String getOriginalAccessKeyId() {
+ return originalAccessKeyId;
+ }
+
+ /**
+ * Get the temporary access key ID (same as owner).
+ */
+ public String getTempAccessKeyId() {
+ return getOwnerId();
+ }
+
+ /**
+ * Optional session policy associated with this STS token, or null/empty if none.
+ */
+ public String getSessionPolicy() {
+ return sessionPolicy;
+ }
+
+ public void setEncryptionKey(byte[] encryptionKey) {
+ this.encryptionKey = encryptionKey.clone();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ if (!super.equals(o)) {
+ return false;
+ }
+
+ final STSTokenIdentifier that = (STSTokenIdentifier) o;
+ return Objects.equals(roleArn, that.roleArn) && Objects.equals(secretAccessKey, that.secretAccessKey) &&
+ Objects.equals(originalAccessKeyId, that.originalAccessKeyId) &&
+ Objects.equals(sessionPolicy, that.sessionPolicy);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ super.hashCode(), roleArn, secretAccessKey, originalAccessKeyId, sessionPolicy);
+ }
+
+ @Override
+ public String toString() {
+ // Intentionally left off secretAccessKey
+ return "STSTokenIdentifier{" + "tempAccessKeyId='" + getOwnerId() + "'" +
+ ", originalAccessKeyId='" + originalAccessKeyId + "', roleArn='" + roleArn + "'" +
+ ", expiry='" + getExpiry() + "', secretKeyId='" + getSecretKeyId() + "'" +
+ ", sessionPolicy='" + sessionPolicy + "'}";
+ }
+}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 799af5d7fa9,2fd6dee16db..1e78813e971
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@@ -161,12 -140,15 +140,10 @@@ public class BucketEndpoint extends Buc
ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow);
} catch (OMException ex) {
- auditReadFailure(s3GAction, ex);
- getMetrics().updateGetBucketFailureStats(startNanos);
+ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
- if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
- // File not found, continue and send normal response with 0 keyCount
- LOG.debug("Key Not found prefix: {}", prefix);
- } else {
- throw ex;
- }
+ handleOMException(ex, bucketName, prefix);
} catch (Exception ex) {
- getMetrics().updateGetBucketFailureStats(startNanos);
- auditReadFailure(s3GAction, ex);
+ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
throw ex;
}
@@@ -247,24 -215,17 +217,14 @@@
addKey(response, next);
count++;
}
- } else {
- addKey(response, next);
- count++;
- }
- if (count == maxKeys) {
- lastKey = next.getName();
- break;
+ if (count == maxKeys) {
+ lastKey = next.getName();
+ break;
+ }
}
+ } catch (RuntimeException ex) {
- getMetrics().updateGetBucketFailureStats(startNanos);
- auditReadFailure(s3GAction, ex);
- if (ex.getCause() instanceof OMException) {
- final OMException omException = (OMException) ex.getCause();
- if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) {
- throw ex;
- }
- handleOMException(omException, bucketName, prefix);
- } else {
- throw ex;
- }
++ handleRuntimeException(ex, context, bucketName, prefix);
}
}
@@@ -517,26 -418,12 +417,40 @@@
OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT);
// initialize handlers
- handlers = new ArrayList<>();
- addHandler(new BucketAclHandler());
- addHandler(new BucketCrudHandler());
- }
-
- private void addHandler(BucketOperationHandler handler) {
- copyDependenciesTo(handler);
- handlers.add(handler);
+ BucketOperationHandler chain = BucketOperationHandlerChain.newBuilder(this)
+ .add(new BucketAclHandler())
+ .add(new ListMultipartUploadsHandler())
+ .add(new BucketCrudHandler())
+ .add(this)
+ .build();
+ handler = new AuditingBucketOperationHandler(chain);
}
+
+ private void handleOMException(OMException ex, String bucketName, String prefix) throws OMException {
+ if (isExpiredToken(ex)) {
+ throw newError(S3ErrorTable.EXPIRED_TOKEN, bucketName, ex);
+ } else if (isAccessDenied(ex)) {
+ throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
+ } else if (ex.getResult() == ResultCodes.FILE_NOT_FOUND) {
+ // File not found, continue and send normal response with 0 keyCount
+ LOG.debug("Key Not found prefix: {}", prefix);
+ } else {
+ throw ex;
+ }
+ }
++
++ private void handleRuntimeException(RuntimeException ex, S3RequestContext context, String bucketName, String prefix)
++ throws OMException {
++ getMetrics().updateGetBucketFailureStats(context.getStartNanos());
++ if (ex.getCause() instanceof OMException) {
++ final OMException omException = (OMException) ex.getCause();
++ if (omException.getResult() == ResultCodes.FILE_NOT_FOUND) {
++ throw ex;
++ }
++ handleOMException(omException, bucketName, prefix);
++ } else {
++ throw ex;
++ }
++ }
++
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 20ad21e23f1,649b14b49cd..0d6fbe1657b
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@@ -697,15 -569,10 +584,15 @@@ public abstract class EndpointBase
AUDIT.logReadFailure(auditMessageForFailure(action, ex).build());
}
- protected boolean isAccessDenied(OMException ex) {
+ protected static boolean isAccessDenied(OMException ex) {
ResultCodes result = ex.getResult();
return result == ResultCodes.PERMISSION_DENIED
- || result == ResultCodes.INVALID_TOKEN;
+ || result == ResultCodes.INVALID_TOKEN
+ || result == ResultCodes.REVOKED_TOKEN;
+ }
+
+ protected boolean isExpiredToken(OMException ex) {
+ return ex.getResult() == ResultCodes.TOKEN_EXPIRED;
}
protected ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket) throws OS3Exception {
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index d97c514f9ae,2021790b332..dd09bff5a41
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@@ -558,14 -561,8 +561,14 @@@ public class ObjectEndpoint extends Obj
if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
// Just return 404 with no content
return Response.status(Status.NOT_FOUND).build();
+ } else if (isExpiredToken(ex)) {
+ throw newError(S3ErrorTable.EXPIRED_TOKEN, keyPath, ex);
+ } else if (isAccessDenied(ex)) {
+ throw newError(S3ErrorTable.ACCESS_DENIED, keyPath, ex);
+ } else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
+ throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else {
- throw ex;
+ throw newError(bucketName, keyPath, ex);
}
} catch (Exception ex) {
auditReadFailure(s3GAction, ex);
@@@ -749,14 -720,17 +726,17 @@@
final String uploadID = queryParams().get(QueryParams.UPLOAD_ID, "");
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.COMPLETE_MULTIPART_UPLOAD;
-- OzoneVolume volume = getVolume();
// Using LinkedHashMap to preserve ordering of parts list.
Map<Integer, String> partsMap = new LinkedHashMap<>();
List<CompleteMultipartUploadRequest.Part> partList =
multipartUploadRequest.getPartList();
+ S3ConditionalRequest.WriteConditions writeConditions =
+ S3ConditionalRequest.parseWriteConditions(getHeaders(), key);
+
OmMultipartUploadCompleteInfo omMultipartUploadCompleteInfo;
try {
++ OzoneVolume volume = getVolume();
OzoneBucket ozoneBucket = volume.getBucket(bucket);
S3Owner.verifyBucketOwnerCondition(getHeaders(), bucket, ozoneBucket.getOwner());
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
index 009e22c42cb,2b6affd3c03..f3ff9f49464
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
@@@ -172,24 -131,20 +157,24 @@@ public class OS3Exception extends Runti
//When we get exception log it, and return exception as xml from actual
// exception data. So, falling back to construct from exception.
- String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
- "<Error>" +
- "<Code>%s</Code>" +
- "<Message>%s</Message>" +
- "<Resource>%s</Resource>" +
- "<RequestId>%s</RequestId>" +
- "</Error>";
- return String.format(formatString, this.getCode(),
- this.getErrorMessage(), this.getResource(),
- this.getRequestId());
+ final StringBuilder builder = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
+ .append("<Error>")
+ .append("<Code>").append(this.getCode()).append("</Code>")
+ .append("<Message>").append(this.getErrorMessage()).append("</Message>")
+ .append("<Resource>").append(this.getResource()).append("</Resource>")
+ .append("<RequestId>").append(this.getRequestId()).append("</RequestId>");
+ if (!Strings.isNullOrEmpty(this.getHostId())) {
+ builder.append("<HostId>").append(this.getHostId()).append("</HostId>");
+ }
+ if (!Strings.isNullOrEmpty(this.getToken0())) {
+ builder.append("<Token-0>").append(this.getToken0()).append("</Token-0>");
+ }
+ builder.append("</Error>");
+ return builder.toString();
}
- /** Create a copy with specific message. */
public OS3Exception withMessage(String message) {
- return new OS3Exception(code, message, httpCode);
+ setErrorMessage(message);
+ return this;
}
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
index 57c870a7eb0,00000000000..0604962cf76
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
@@@ -1,160 -1,0 +1,173 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents exceptions raised from Ozone STS service.
+ */
+public class OSTSException extends OS3Exception {
+ private static final Logger LOG = LoggerFactory.getLogger(OSTSException.class);
+ private static final ObjectMapper MAPPER;
+ private static final String AWS_FAULT_NS = "http://webservices.amazon.com/AWSFault/2005-15-09";
+ private static final String STS_NS = "https://sts.amazonaws.com/doc/2011-06-15/";
+ private static final String INVALID_ACTION = "InvalidAction";
+
+ static {
+ MAPPER = new XmlMapper();
+ MAPPER.registerModule(new JaxbAnnotationModule());
+ MAPPER.enable(SerializationFeature.INDENT_OUTPUT);
+ }
+
+ private String type = "Sender";
+
- public OSTSException(String codeVal, String messageVal, int httpCode) {
- super(codeVal, messageVal, httpCode);
++ public OSTSException(S3ErrorTable error) {
++ super(error, null, null);
+ }
+
- public OSTSException(String codeVal, String messageVal, int httpCode, String typeVal) {
- this(codeVal, messageVal, httpCode);
++ public OSTSException(S3ErrorTable error, Exception cause) {
++ super(error, cause, null);
++ }
++
++ @Override
++ public OSTSException withMessage(String message) {
++ super.withMessage(message);
++ return this;
++ }
++
++ /**
++ * STS fault party (e.g. {@code Sender} vs {@code Receiver}); defaults to {@code Sender}.
++ */
++ public OSTSException withType(String typeVal) {
+ this.type = typeVal;
++ return this;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String toXml() {
+ try {
+ final ErrorResponse response = new ErrorResponse(this);
+ final String val = MAPPER.writeValueAsString(response);
+ LOG.debug("toXml val is {}", val);
+ return val; // STS error responses don't have prolog <?xml version="1.0" encoding="UTF-8"?>
+ } catch (Exception ex) {
+ LOG.error("Exception occurred", ex);
+ // Fallback
+ final String namespace = INVALID_ACTION.equals(getCode()) ? AWS_FAULT_NS : STS_NS;
+
+ // STS error responses don't have prolog <?xml version="1.0" encoding="UTF-8"?>
+ final StringBuilder builder = new StringBuilder();
+ builder.append("<ErrorResponse xmlns=\"").append(namespace).append("\">\n")
+ .append(" <Error>\n")
+ .append(" <Type>").append(getType()).append("</Type>\n")
+ .append(" <Code>").append(getCode()).append("</Code>\n")
+ .append(" <Message>").append(getErrorMessage()).append("</Message>\n")
+ .append(" </Error>\n")
+ .append(" <RequestId>").append(getRequestId()).append("</RequestId>\n")
+ .append("</ErrorResponse>");
+ return builder.toString();
+ }
+ }
+
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlRootElement(name = "ErrorResponse")
+ private static class ErrorResponse {
+
+ @XmlAttribute
+ private String xmlns;
+
+ @XmlElement(name = "Error")
+ private ErrorDetails error;
+
+ @XmlElement(name = "RequestId")
+ private String requestId;
+
+ ErrorResponse() {
+ }
+
+ ErrorResponse(OSTSException ex) {
+ this.xmlns = INVALID_ACTION.equals(ex.getCode()) ? AWS_FAULT_NS : STS_NS;
+ this.error = new ErrorDetails(ex.getType(), ex.getCode(), ex.getErrorMessage());
+ this.requestId = ex.getRequestId();
+ }
+
+ public String getXmlns() {
+ return xmlns;
+ }
+
+ public ErrorDetails getError() {
+ return error;
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+ }
+
+ @XmlAccessorType(XmlAccessType.FIELD)
+ private static class ErrorDetails {
+ @XmlElement(name = "Type")
+ private String type;
+
+ @XmlElement(name = "Code")
+ private String code;
+
+ @XmlElement(name = "Message")
+ private String message;
+
+ ErrorDetails() {
+ }
+
+ ErrorDetails(String type, String code, String message) {
+ this.type = type;
+ this.code = code;
+ this.message = message;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+ }
+}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index f315012a976,022dc08949e..ee269b7542a
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@@ -32,122 -33,118 +33,122 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
/**
-- * This class represents errors from Ozone S3 service.
++ * This class represents errors from Ozone S3 service and STS-compatible errors
++ * raised by the S3 gateway STS endpoint.
* This class needs to be updated to add new errors when required.
*/
- public final class S3ErrorTable {
+ public enum S3ErrorTable {
- private static final Logger LOG = LoggerFactory.getLogger(
- S3ErrorTable.class);
+ INVALID_URI("InvalidURI",
+ "Couldn't parse the specified URI.", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_URI = new OS3Exception("InvalidURI",
- "Couldn't parse the specified URI.", HTTP_BAD_REQUEST);
+ NO_SUCH_VOLUME("NoSuchVolume",
+ "The specified volume does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception NO_SUCH_VOLUME = new OS3Exception(
- "NoSuchVolume", "The specified volume does not exist", HTTP_NOT_FOUND);
+ NO_SUCH_BUCKET("NoSuchBucket",
+ "The specified bucket does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception NO_SUCH_BUCKET = new OS3Exception(
- "NoSuchBucket", "The specified bucket does not exist", HTTP_NOT_FOUND);
+ AUTH_PROTOCOL_NOT_SUPPORTED("AuthProtocolNotSupported",
+ "Auth protocol used for this request is not supported.", HTTP_BAD_REQUEST),
- public static final OS3Exception AUTH_PROTOCOL_NOT_SUPPORTED =
- new OS3Exception("AuthProtocolNotSupported", "Auth protocol used for" +
- " this request is not supported.", HTTP_BAD_REQUEST);
-
- public static final OS3Exception S3_AUTHINFO_CREATION_ERROR =
- new OS3Exception("InvalidRequest", "Error creating s3 auth info. " +
+ S3_AUTHINFO_CREATION_ERROR("InvalidRequest", "Error creating s3 auth info. " +
"The request may not be signed using AWS V4 signing algorithm," +
- " or might be invalid", HTTP_FORBIDDEN);
+ " or might be invalid", HTTP_FORBIDDEN),
- public static final OS3Exception BUCKET_NOT_EMPTY = new OS3Exception(
+ BUCKET_NOT_EMPTY(
"BucketNotEmpty", "The bucket you tried to delete is not empty. " +
"If you are using --force option to delete all objects in the bucket, " +
"please ensure that the bucket layout is OBJECT_STORE or " +
"that the bucket is completely empty before delete.",
- HTTP_CONFLICT);
+ HTTP_CONFLICT),
- public static final OS3Exception MALFORMED_HEADER = new OS3Exception(
+ MALFORMED_HEADER(
"AuthorizationHeaderMalformed", "The authorization header you provided " +
- "is invalid.", HTTP_BAD_REQUEST);
+ "is invalid.", HTTP_BAD_REQUEST),
- public static final OS3Exception NO_SUCH_KEY = new OS3Exception(
- "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND);
+ NO_SUCH_KEY(
+ "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND),
- public static final OS3Exception INVALID_ARGUMENT = new OS3Exception(
- "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST);
+ INVALID_ARGUMENT(
+ "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_REQUEST = new OS3Exception(
- "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
+ INVALID_REQUEST(
+ "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_RANGE = new OS3Exception(
+ INVALID_RANGE(
"InvalidRange", "The requested range is not satisfiable",
- RANGE_NOT_SATISFIABLE);
+ RANGE_NOT_SATISFIABLE),
- public static final OS3Exception NO_SUCH_UPLOAD = new OS3Exception(
+ NO_SUCH_UPLOAD(
"NoSuchUpload", "The specified multipart upload does not exist. The " +
"upload ID might be invalid, or the multipart upload might have " +
- "been aborted or completed.", HTTP_NOT_FOUND);
+ "been aborted or completed.", HTTP_NOT_FOUND),
- public static final OS3Exception INVALID_BUCKET_NAME = new OS3Exception(
+ INVALID_BUCKET_NAME(
"InvalidBucketName", "The specified bucket is not valid.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_PART = new OS3Exception(
+ INVALID_PART(
"InvalidPart", "One or more of the specified parts could not be found." +
" The part might not have been uploaded, or the specified entity " +
- "tag might not have matched the part's entity tag.", HTTP_BAD_REQUEST);
+ "tag might not have matched the part's entity tag.", HTTP_BAD_REQUEST),
- public static final OS3Exception INVALID_PART_ORDER = new OS3Exception(
+ INVALID_PART_ORDER(
"InvalidPartOrder", "The list of parts was not in ascending order. The " +
"parts list must be specified in order by part number.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception ENTITY_TOO_SMALL = new OS3Exception(
+ ENTITY_TOO_SMALL(
"EntityTooSmall", "Your proposed upload is smaller than the minimum " +
"allowed object size. Each part must be at least 5 MB in size, except " +
- "the last part.", HTTP_BAD_REQUEST);
+ "the last part.", HTTP_BAD_REQUEST),
- public static final OS3Exception INTERNAL_ERROR = new OS3Exception(
+ INTERNAL_ERROR(
"InternalError", "We encountered an internal error. Please try again.",
- HTTP_INTERNAL_ERROR);
+ HTTP_INTERNAL_ERROR),
- public static final OS3Exception ACCESS_DENIED = new OS3Exception(
+ ACCESS_DENIED(
"AccessDenied", "User doesn't have the right to access this " +
- "resource.", HTTP_FORBIDDEN);
+ "resource.", HTTP_FORBIDDEN),
- public static final OS3Exception EXPIRED_TOKEN = new OS3Exception(
- "ExpiredToken", "The provided token has expired.", HTTP_BAD_REQUEST);
++ EXPIRED_TOKEN(
++ "ExpiredToken", "The provided token has expired.", HTTP_BAD_REQUEST),
+
- public static final OS3Exception PRECOND_FAILED = new OS3Exception(
+ PRECOND_FAILED(
"PreconditionFailed", "At least one of the pre-conditions you " +
- "specified did not hold", HTTP_PRECON_FAILED);
+ "specified did not hold", HTTP_PRECON_FAILED),
+
+ CONDITIONAL_REQUEST_CONFLICT("ConditionalRequestConflict",
+ "A conflicting conditional operation occurred. Retry the request.",
+ HTTP_CONFLICT),
- public static final OS3Exception NOT_IMPLEMENTED = new OS3Exception(
+ NOT_IMPLEMENTED(
"NotImplemented", "This part of feature is not implemented yet.",
- HTTP_NOT_IMPLEMENTED);
+ HTTP_NOT_IMPLEMENTED),
- public static final OS3Exception NO_OVERWRITE = new OS3Exception(
- "Conflict", "Cannot overwrite file with directory", HTTP_CONFLICT);
+ NO_OVERWRITE(
+ "Conflict", "Cannot overwrite file with directory", HTTP_CONFLICT),
- public static final OS3Exception METADATA_TOO_LARGE = new OS3Exception(
+ METADATA_TOO_LARGE(
"MetadataTooLarge", "Illegal user defined metadata. Combined size " +
"exceeds the maximum allowed metadata size of " +
- S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB + "KB", HTTP_BAD_REQUEST);
+ S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB + "KB", HTTP_BAD_REQUEST),
- public static final OS3Exception BUCKET_ALREADY_EXISTS = new OS3Exception(
+ BUCKET_ALREADY_EXISTS(
"BucketAlreadyExists", "The requested bucket name is not available" +
- " as it already exists.", HTTP_CONFLICT);
+ " as it already exists.", HTTP_CONFLICT),
- public static final OS3Exception INVALID_TAG = new OS3Exception(
- "InvalidTag", "Your request contains tag input that is not valid.", HTTP_BAD_REQUEST);
+ INVALID_TAG(
+ "InvalidTag", "Your request contains tag input that is not valid.", HTTP_BAD_REQUEST),
- public static final OS3Exception NO_SUCH_TAG_SET = new OS3Exception(
- "NoSuchTagSet", "The specified tag does not exist.", HTTP_NOT_FOUND);
+ NO_SUCH_TAG_SET(
+ "NoSuchTagSet", "The specified tag does not exist.", HTTP_NOT_FOUND),
- public static final OS3Exception MALFORMED_XML = new OS3Exception(
+ MALFORMED_XML(
"MalformedXML", "The XML you provided was not well-formed or did not " +
- "validate against our published schema", HTTP_BAD_REQUEST);
+ "validate against our published schema", HTTP_BAD_REQUEST),
- public static final OS3Exception QUOTA_EXCEEDED = new OS3Exception(
+ QUOTA_EXCEEDED(
"QuotaExceeded", "The quota has been exceeded. " +
"Please review your disk space or namespace usage and adjust accordingly.",
HTTP_FORBIDDEN
@@@ -157,56 -154,128 +158,166 @@@
"InvalidStorageClass", "The storage class that you specified is not valid. " +
"Provide a supported storage class[STANDARD|REDUCED_REDUNDANCY|STANDARD_IA] or " +
"a valid custom EC storage config for if using STANDARD_IA.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
- public static final OS3Exception BUCKET_OWNER_MISMATCH = new OS3Exception(
+ BUCKET_OWNER_MISMATCH(
"Access Denied", "User doesn't have permission to access this resource due to a " +
- "bucket ownership mismatch.", HTTP_FORBIDDEN);
+ "bucket ownership mismatch.", HTTP_FORBIDDEN),
- public static final OS3Exception PAYLOAD_TOO_LARGE = new OS3Exception(
- "PayloadTooLarge", "Your request body size was too large.", HTTP_BAD_REQUEST
- );
++ PAYLOAD_TOO_LARGE(
++ "PayloadTooLarge", "Your request body size was too large.", HTTP_BAD_REQUEST),
+
- public static final OS3Exception X_AMZ_CONTENT_SHA256_MISMATCH = new OS3Exception(
+ X_AMZ_CONTENT_SHA256_MISMATCH(
"XAmzContentSHA256Mismatch", "The provided 'x-amz-content-sha256' header does " +
- "not match the computed hash.", HTTP_BAD_REQUEST);
+ "not match the computed hash.", HTTP_BAD_REQUEST),
- public static final OS3Exception BAD_DIGEST = new OS3Exception(
+ BAD_DIGEST(
"BadDigest", "The Content-MD5 or checksum value that you specified did not match what the server received.",
- HTTP_BAD_REQUEST);
+ HTTP_BAD_REQUEST),
+
+ INVALID_DIGEST(
- "InvalidDigest", "The Content-MD5 you specified is not valid.", HTTP_BAD_REQUEST);
++ "InvalidDigest", "The Content-MD5 you specified is not valid.", HTTP_BAD_REQUEST),
++
++ /** STS: Code {@code ValidationError}; message is usually overridden via {@link OS3Exception#withMessage}. */
++ STS_VALIDATION_ERROR(
++ "ValidationError", "A validation error occurred.", HTTP_BAD_REQUEST),
++
++ /**
++ * STS: Code {@code InvalidAction} with HTTP 400 (unknown action or unsupported API version).
++ * For HTTP 501 unsupported-but-known operations use {@link #STS_INVALID_ACTION_NOT_IMPLEMENTED}.
++ */
++ STS_INVALID_ACTION(
++ "InvalidAction", "Could not find operation.", HTTP_BAD_REQUEST),
++
++ /** STS: Code {@code InvalidAction} with HTTP 501 (operation known but not implemented). */
++ STS_INVALID_ACTION_NOT_IMPLEMENTED(
++ "InvalidAction", "Operation is not supported yet.", HTTP_NOT_IMPLEMENTED),
++
++ /** STS: Code {@code InternalFailure}; distinct from {@link #INTERNAL_ERROR} ({@code InternalError}). */
++ STS_INTERNAL_FAILURE(
++ "InternalFailure", "An internal error has occurred.", HTTP_INTERNAL_ERROR),
+
- private static Function<Exception, OS3Exception> generateInternalError =
- e -> new OS3Exception("InternalError", e.getMessage(), HTTP_INTERNAL_ERROR);
++ /** STS: Code {@code InvalidClientTokenId}. */
++ STS_INVALID_CLIENT_TOKEN_ID(
++ "InvalidClientTokenId",
++ "The security token included in the request is invalid.", HTTP_FORBIDDEN),
+
- private S3ErrorTable() {
- //No one should construct this object.
++ /** STS: Code {@code UnsupportedOperation}; distinct from {@link #NOT_IMPLEMENTED} ({@code NotImplemented}). */
++ STS_UNSUPPORTED_OPERATION(
++ "UnsupportedOperation", "This operation is not supported.", HTTP_NOT_IMPLEMENTED),
++
++ /** STS: Code {@code MalformedPolicyDocument}; message is usually overridden via {@link OS3Exception#withMessage}. */
++ STS_MALFORMED_POLICY_DOCUMENT(
++ "MalformedPolicyDocument", "Policy document is malformed.", HTTP_BAD_REQUEST);
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3ErrorTable.class);
+
+ private final String code;
+ private final String message;
+ private final int httpCode;
+
+ S3ErrorTable(String code, String message, int httpCode) {
+ this.code = code;
+ this.message = message;
+ this.httpCode = httpCode;
}
- public static OS3Exception newError(OS3Exception e, String resource) {
- return newError(e, resource, null);
+ public String getCode() {
+ return code;
}
- /**
- * Create a new instance of Error.
- * @param e Error Template
- * @param resource Resource associated with this exception
- * @param ex the original exception, may be null
- * @return creates a new instance of error based on the template
- */
- public static OS3Exception newError(OS3Exception e, String resource,
- Exception ex) {
- OS3Exception err = new OS3Exception(e.getCode(), e.getErrorMessage(),
- e.getHttpCode());
- err.setResource(resource);
- if (e.getHttpCode() == HTTP_INTERNAL_ERROR) {
- LOG.error("Internal Error: {}", err.toXml(), ex);
- } else if (LOG.isDebugEnabled()) {
- LOG.debug(err.toXml(), ex);
+ public String getErrorMessage() {
+ return message;
+ }
+
+ public int getHttpCode() {
+ return httpCode;
+ }
+
+ /** Converts result code of {@code OMException} to {@code S3ErrorTable}, which
+ * should be thrown via {@link #newError(S3ErrorTable, String, Exception)}. */
+ public static S3ErrorTable translateResultCode(OMException ex) {
+ switch (ex.getResult()) {
+ case ACCESS_DENIED:
+ case INVALID_TOKEN:
+ case PERMISSION_DENIED:
++ case REVOKED_TOKEN:
+ return ACCESS_DENIED;
+ case ATOMIC_WRITE_CONFLICT:
+ return CONDITIONAL_REQUEST_CONFLICT;
+ case BUCKET_ALREADY_EXISTS:
+ return BUCKET_ALREADY_EXISTS;
+ case BUCKET_NOT_EMPTY:
+ return BUCKET_NOT_EMPTY;
+ case BUCKET_NOT_FOUND:
+ case VOLUME_NOT_FOUND:
+ return NO_SUCH_BUCKET;
+ case ENTITY_TOO_SMALL:
+ return ENTITY_TOO_SMALL;
+ case ETAG_MISMATCH:
+ case ETAG_NOT_AVAILABLE:
+ case KEY_ALREADY_EXISTS:
+ return PRECOND_FAILED;
++ case TOKEN_EXPIRED:
++ return EXPIRED_TOKEN;
+ case FILE_ALREADY_EXISTS:
+ return NO_OVERWRITE;
+ case INVALID_BUCKET_NAME:
+ return INVALID_BUCKET_NAME;
+ case INVALID_PART:
+ return INVALID_PART;
+ case INVALID_PART_ORDER:
+ return INVALID_PART_ORDER;
+ case INVALID_REQUEST:
+ return INVALID_REQUEST;
+ case KEY_NOT_FOUND:
+ return NO_SUCH_KEY;
+ case NOT_SUPPORTED_OPERATION:
+ return NOT_IMPLEMENTED;
+ case QUOTA_EXCEEDED:
+ return QUOTA_EXCEEDED;
+ default:
+ return INTERNAL_ERROR;
}
- return err;
}
- public static OS3Exception getInternalError(Exception e) {
- return generateInternalError.apply(e);
+ /** Same as {@link #newError(String, OMException)}, but uses {@code bucket} as the {@code resource}
+ * in case of {@link #NO_SUCH_BUCKET} error. */
+ public static OS3Exception newError(String bucket, String resource, OMException cause) {
+ S3ErrorTable errorCode = translateResultCode(cause);
+ return new OS3Exception(errorCode, cause, NO_SUCH_BUCKET == errorCode ? bucket : resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link OMException} and {@code resource}. */
+ public static OS3Exception newError(@Nullable String resource, OMException cause) {
+ return new OS3Exception(translateResultCode(cause), cause, resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable}. */
+ public static OS3Exception newError(S3ErrorTable errorCode) {
+ return new OS3Exception(errorCode, null, null);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable} and {@code resource}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, @Nullable String resource) {
+ return new OS3Exception(errorCode, null, resource);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable} and {@code cause}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, Exception cause) {
+ return new OS3Exception(errorCode, cause, null);
+ }
+
+ /** Creates new {@link OS3Exception} for {@link S3ErrorTable}, {@code resource} and {@code cause}. */
+ public static OS3Exception newError(S3ErrorTable errorCode, @Nullable String resource, Exception cause) {
+ return new OS3Exception(errorCode, cause, resource);
+ }
+
+ static void log(OS3Exception err) {
+ if (err.getHttpCode() == HTTP_INTERNAL_ERROR) {
+ LOG.error("Internal Error: {}", err.toXml(), err.getCause());
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug(err.toXml(), err.getCause());
+ }
}
}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
index 343c341ee4f,02067537495..16acd86ec6f
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
@@@ -123,75 -96,6 +123,75 @@@ public class AWSSignatureProcessor impl
return signatureInfo;
}
+ private String extractSessionToken(LowerCaseKeyStringMap headers) {
+ // Header-based token
+ final String headerToken = headers.get(X_AMZ_SECURITY_TOKEN);
+ if (headerToken != null && !headerToken.isEmpty()) {
+ return headerToken;
+ }
+
+ // Query-based token - this would be used for presigned URLs
+ final MultivaluedMap<String, String> queryParams = context.getUriInfo().getQueryParameters();
+ if (queryParams == null) {
+ return null;
+ }
+ for (Map.Entry<String, List<String>> entry : queryParams.entrySet()) {
+ final String key = entry.getKey();
+ if (Strings.isNullOrEmpty(key)) {
+ continue;
+ }
+ if (key.compareToIgnoreCase(X_AMZ_SECURITY_TOKEN) == 0) {
+ final List<String> values = entry.getValue();
+ if (values != null && !values.isEmpty()) {
+ return values.get(0);
+ }
+ }
+ }
+ return null;
+ }
+
+ private String getPayloadHash(Map<String, String> headers, SignatureInfo signatureInfo)
+ throws OS3Exception, NoSuchAlgorithmException, IOException {
+ if (signatureInfo.getVersion() == Version.V2) {
- throw S3_AUTHINFO_CREATION_ERROR;
++ throw S3ErrorTable.newError(S3_AUTHINFO_CREATION_ERROR);
+ }
+ if (signatureInfo.getService().equals("s3")) {
+ if (!signatureInfo.isSignPayload()) {
+ // According to AWS Signature V4 documentation using Query Parameters
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+ return UNSIGNED_PAYLOAD;
+ }
+ String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256);
+ // According to AWS Signature V4 documentation using Authorization Header
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ // The x-amz-content-sha256 header is required
+ // for all AWS Signature Version 4 requests using Authorization header.
+ if (contentSignatureHeaderValue == null) {
+ LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
+ + " header for signed payload");
- throw S3_AUTHINFO_CREATION_ERROR;
++ throw S3ErrorTable.newError(S3_AUTHINFO_CREATION_ERROR);
+ }
+ // Simply return the header value of x-amz-content-sha256 as the payload hash
+ // These are the possible cases:
+ // 1. Actual payload checksum for single chunk upload
+ // 2. Unsigned payloads for multiple chunks upload
+ // - UNSIGNED-PAYLOAD
+ // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
+ // 3. Signed payloads for multiple chunks upload
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+ return contentSignatureHeaderValue;
+ }
+ // For STS payload hash is calculated over the body
+ InputStream in = context.getEntityStream();
+ byte[] body = readAllBytes(in);
+ String payloadHash = Hex.encode(MessageDigest.getInstance("SHA-256").digest(body));
+ context.setEntityStream(new ByteArrayInputStream(body));
+ return payloadHash;
+ }
+
private AuditMessage buildAuthFailureMessage(MalformedResourceException e) {
AuditMessage message = new AuditMessage.Builder()
.forOperation(AuthOperation.fromContext(context))
@@@ -203,22 -107,6 +203,21 @@@
return message;
}
+ private byte[] readAllBytes(InputStream in) throws OS3Exception, IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ byte[] chunk = new byte[8192];
+ int totalRead = 0;
+ int n;
+ while ((n = in.read(chunk)) != -1) {
+ if (totalRead + n > OZONE_S3G_STS_PAYLOAD_HASH_MAX_VALUE) {
- throw new OSTSException(
- PAYLOAD_TOO_LARGE.getCode(), PAYLOAD_TOO_LARGE.getErrorMessage(), PAYLOAD_TOO_LARGE.getHttpCode());
++ throw new OSTSException(PAYLOAD_TOO_LARGE);
+ }
+ buffer.write(chunk, 0, n);
+ totalRead += n;
+ }
+ return buffer.toByteArray();
+ }
+
@VisibleForTesting
public void setContext(ContainerRequestContext context) {
this.context = context;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index 9be7f7d6752,f308363133d..3eab4b1f33e
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@@ -19,6 -19,8 +19,7 @@@ package org.apache.hadoop.ozone.s3.sign
import static java.time.temporal.ChronoUnit.SECONDS;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR;
+ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.UNSIGNED_PAYLOAD;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import com.google.common.annotations.VisibleForTesting;
@@@ -173,15 -176,12 +175,15 @@@ public final class StringToSignProduce
StringBuilder canonicalHeaders = new StringBuilder();
for (String header : StringUtils.split(signedHeaders, ';')) {
- canonicalHeaders.append(header.toLowerCase());
- canonicalHeaders.append(':');
+ canonicalHeaders.append(header.toLowerCase())
+ .append(':');
if (headers.containsKey(header)) {
String headerValue = headers.get(header);
+ if (header.equals("content-type")) {
+ headerValue = headerValue.toLowerCase();
+ }
- canonicalHeaders.append(headerValue);
- canonicalHeaders.append(NEWLINE);
+ canonicalHeaders.append(headerValue)
+ .append(NEWLINE);
// Set for testing purpose only to skip date and host validation.
try {
@@@ -207,7 -210,38 +209,7 @@@
+ signedHeaders + NEWLINE
+ payloadHash;
}
-
- private static String getPayloadHash(Map<String, String> headers, boolean isUsingQueryParameter)
- throws OS3Exception {
- if (isUsingQueryParameter) {
- // According to AWS Signature V4 documentation using Query Parameters
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
- return UNSIGNED_PAYLOAD;
- }
- String contentSignatureHeaderValue = headers.get(X_AMZ_CONTENT_SHA256);
- // According to AWS Signature V4 documentation using Authorization Header
- // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
- // The x-amz-content-sha256 header is required
- // for all AWS Signature Version 4 requests using Authorization header.
- if (contentSignatureHeaderValue == null) {
- LOG.error("The request must include " + X_AMZ_CONTENT_SHA256
- + " header for signed payload");
- throw newError(S3_AUTHINFO_CREATION_ERROR);
- }
- // Simply return the header value of x-amz-content-sha256 as the payload hash
- // These are the possible cases:
- // 1. Actual payload checksum for single chunk upload
- // 2. Unsigned payloads for multiple chunks upload
- // - UNSIGNED-PAYLOAD
- // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
- // 3. Signed payloads for multiple chunks upload
- // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
- // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
- // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
- // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
- return contentSignatureHeaderValue;
- }
--
++
/**
* String join that also works with empty strings.
*
@@@ -324,11 -357,12 +325,11 @@@
*/
private static void validateCanonicalHeaders(
String canonicalHeaders,
- Map<String, String> headers,
- Boolean unsignedPaylod
- ) throws OS3Exception {
+ Map<String, String> headers)
+ throws OS3Exception {
if (!canonicalHeaders.contains(HOST + ":")) {
LOG.error("The SignedHeaders list must include HTTP Host header");
- throw S3_AUTHINFO_CREATION_ERROR;
+ throw newError(S3_AUTHINFO_CREATION_ERROR);
}
for (String header : headers.keySet().stream()
.filter(s -> s.startsWith("x-amz-"))
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
index 091f8851fa3,00000000000..1cc92230149
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
@@@ -1,354 -1,0 +1,344 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3sts;
+
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
- import static javax.ws.rs.core.Response.Status.FORBIDDEN;
- import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
- import static javax.ws.rs.core.Response.Status.NOT_IMPLEMENTED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INTERNAL_FAILURE;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_ACTION;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_ACTION_NOT_IMPLEMENTED;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_INVALID_CLIENT_TOKEN_ID;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_MALFORMED_POLICY_DOCUMENT;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_UNSUPPORTED_OPERATION;
++import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.STS_VALIDATION_ERROR;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.audit.S3GAction;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo;
+import org.apache.hadoop.ozone.om.helpers.AwsRoleArnValidator;
+import org.apache.hadoop.ozone.om.helpers.S3STSUtils;
+import org.apache.hadoop.ozone.s3.RequestIdentifier;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.OSTSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * AWS STS (Security Token Service) compatible endpoint for Ozone S3 Gateway.
+ * <p>
+ * This endpoint provides temporary security credentials compatible with
+ * AWS STS API, exposed on the port 9880 or 9881.
+ * <p>
+ * Currently supports only AssumeRole operation. Other STS operations will
+ * return appropriate error responses.
+ *
+ * @see <a href="https://docs.aws.amazon.com/STS/latest/APIReference/">AWS STS API Reference</a>
+ */
+@Path("/")
+@S3STSEnabled
+public class S3STSEndpoint extends S3STSEndpointBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3STSEndpoint.class);
+
+ // STS API constants
+ private static final String ASSUME_ROLE_ACTION = "AssumeRole";
+ private static final String GET_SESSION_TOKEN_ACTION = "GetSessionToken";
+ private static final String ASSUME_ROLE_WITH_SAML_ACTION = "AssumeRoleWithSAML";
+ private static final String ASSUME_ROLE_WITH_WEB_IDENTITY_ACTION = "AssumeRoleWithWebIdentity";
+ private static final String GET_CALLER_IDENTITY_ACTION = "GetCallerIdentity";
+ private static final String DECODE_AUTHORIZATION_MESSAGE_ACTION = "DecodeAuthorizationMessage";
+ private static final String GET_ACCESS_KEY_INFO_ACTION = "GetAccessKeyInfo";
+
+ private static final String EXPECTED_VERSION = "2011-06-15";
- private static final String VALIDATION_ERROR = "ValidationError";
- private static final String INVALID_ACTION = "InvalidAction";
- private static final String INTERNAL_FAILURE = "InternalFailure";
- private static final String ACCESS_DENIED = "AccessDenied";
- private static final String INVALID_CLIENT_TOKEN_ID = "InvalidClientTokenId";
- private static final String UNSUPPORTED_OPERATION = "UnsupportedOperation";
- private static final String MALFORMED_POLICY_DOCUMENT = "MalformedPolicyDocument";
+
+ // JAXBContext is relatively expensive to create and is threadsafe, so cache and reuse
+ private static final JAXBContext JAXB_CONTEXT;
+
+ static {
+ try {
+ JAXB_CONTEXT = JAXBContext.newInstance(S3AssumeRoleResponseXml.class);
+ } catch (JAXBException e) {
+ throw new RuntimeException("Failed to initialize JAXBContext: " + e, e);
+ }
+ }
+
+ @Inject
+ private RequestIdentifier requestIdentifier;
+
+ @VisibleForTesting
+ public void setRequestIdentifier(RequestIdentifier requestIdentifier) {
+ this.requestIdentifier = requestIdentifier;
+ }
+
+ /**
+ * STS endpoint that handles GET requests with query parameters.
+ * AWS STS supports both GET and POST requests.
+ *
+ * @param action The STS action to perform (AssumeRole, GetSessionToken, etc.)
+ * @param roleArn The ARN of the role to assume (for AssumeRole)
+ * @param roleSessionName Session name for the role (for AssumeRole)
+ * @param durationSeconds Duration of the token validity in seconds
+ * @param version AWS STS API version (should be "2011-06-15")
+ * @return Response containing STS response XML or error
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_XML)
+ public Response get(
+ @QueryParam("Action") String action,
+ @QueryParam("RoleArn") String roleArn,
+ @QueryParam("RoleSessionName") String roleSessionName,
+ @QueryParam("DurationSeconds") Integer durationSeconds,
+ @QueryParam("Version") String version,
+ @QueryParam("Policy") String awsIamSessionPolicy) throws OS3Exception {
+
+ return handleSTSRequest(action, roleArn, roleSessionName, durationSeconds, version, awsIamSessionPolicy);
+ }
+
+ /**
+ * STS endpoint that handles POST requests with form data.
+ * AWS STS typically uses POST requests with form-encoded parameters.
+ *
+ * @param action The STS action to perform
+ * @param roleArn The ARN of the role to assume
+ * @param roleSessionName Session name for the role
+ * @param durationSeconds Duration of the token validity
+ * @param version AWS STS API version
+ * @return Response containing STS response XML or error
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_XML)
+ public Response post(
+ @FormParam("Action") String action,
+ @FormParam("RoleArn") String roleArn,
+ @FormParam("RoleSessionName") String roleSessionName,
+ @FormParam("DurationSeconds") Integer durationSeconds,
+ @FormParam("Version") String version,
+ @FormParam("Policy") String awsIamSessionPolicy) throws OS3Exception {
+
+ return handleSTSRequest(action, roleArn, roleSessionName, durationSeconds, version, awsIamSessionPolicy);
+ }
+
+ private Response handleSTSRequest(String action, String roleArn, String roleSessionName,
+ Integer durationSeconds, String version, String awsIamSessionPolicy) throws OS3Exception {
+ final String requestId = requestIdentifier.getRequestId();
+ // NOTE: invalid, missing or unsupported actions are not added to the audit log
+ try {
+ if (action == null) {
+ // Amazon STS has a different structure for the XML error response when the action is missing
+ return Response.status(BAD_REQUEST)
+ .entity("<UnknownOperationException/>")
+ .type(MediaType.APPLICATION_XML)
+ .build();
+ }
+
+ switch (action) {
+ case ASSUME_ROLE_ACTION:
+ return handleAssumeRole(roleArn, roleSessionName, durationSeconds, awsIamSessionPolicy, version, requestId);
+ // These operations are not supported yet
+ case GET_SESSION_TOKEN_ACTION:
+ case ASSUME_ROLE_WITH_SAML_ACTION:
+ case ASSUME_ROLE_WITH_WEB_IDENTITY_ACTION:
+ case GET_CALLER_IDENTITY_ACTION:
+ case DECODE_AUTHORIZATION_MESSAGE_ACTION:
+ case GET_ACCESS_KEY_INFO_ACTION:
- throw new OSTSException(
- INVALID_ACTION, "Operation " + action + " is not supported yet.", NOT_IMPLEMENTED.getStatusCode());
++ throw new OSTSException(STS_INVALID_ACTION_NOT_IMPLEMENTED)
++ .withMessage("Operation " + action + " is not supported yet.");
+ default:
- throw new OSTSException(
- INVALID_ACTION, "Could not find operation " + action + " for version " +
- (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version),
- BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_INVALID_ACTION)
++ .withMessage("Could not find operation " + action + " for version " +
++ (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version));
+ }
+ } catch (OSTSException e) {
+ throw e;
+ } catch (Exception ex) {
+ LOG.error("Unexpected error during STS request", ex);
- throw new OSTSException(
- INTERNAL_FAILURE, "An internal error has occurred.", INTERNAL_SERVER_ERROR.getStatusCode(), "Receiver");
++ throw new OSTSException(STS_INTERNAL_FAILURE, ex).withType("Receiver");
+ }
+ }
+
+ private Response handleAssumeRole(String roleArn, String roleSessionName, Integer durationSeconds,
+ String awsIamSessionPolicy, String version, String requestId) throws OSTSException {
+ final String action = "AssumeRole";
+ final Map<String, String> auditParams = getAuditParameters();
+ S3STSUtils.addAssumeRoleAuditParams(
+ auditParams, roleArn, roleSessionName, awsIamSessionPolicy,
+ durationSeconds == null ? S3STSUtils.DEFAULT_DURATION_SECONDS : durationSeconds,
+ requestId);
+
+ // Validate parameters
+ if (version == null || !version.equals(EXPECTED_VERSION)) {
- final OSTSException exception = new OSTSException(
- INVALID_ACTION, "Could not find operation " + action + " for version " +
- (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version),
- BAD_REQUEST.getStatusCode());
++ final OSTSException exception = new OSTSException(STS_INVALID_ACTION)
++ .withMessage("Could not find operation " + action + " for version " +
++ (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version));
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, exception));
+ throw exception;
+ }
+
+ final Set<String> validationErrors = new HashSet<>();
+ int duration = durationSeconds == null ? S3STSUtils.DEFAULT_DURATION_SECONDS : durationSeconds;
+ try {
+ duration = S3STSUtils.validateDuration(durationSeconds);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ AwsRoleArnValidator.validateAndExtractRoleNameFromArn(roleArn);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ S3STSUtils.validateRoleSessionName(roleSessionName);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ try {
+ if (LOG.isDebugEnabled() && StringUtils.isNotEmpty(awsIamSessionPolicy)) {
+ LOG.debug(
+ "AssumeRole requestId={} received Policy(len={}): {}", requestId, awsIamSessionPolicy.length(),
+ awsIamSessionPolicy);
+ }
+ S3STSUtils.validateSessionPolicy(awsIamSessionPolicy);
+ } catch (OMException e) {
+ validationErrors.add(e.getMessage());
+ }
+
+ final int numValidationErrors = validationErrors.size();
+ if (numValidationErrors > 0) {
+ //noinspection StringBufferReplaceableByString
- final StringBuilder builder = new StringBuilder();
- builder.append(numValidationErrors);
- builder.append(" validation ");
- builder.append(numValidationErrors > 1 ? "errors detected: " : "error detected: ");
- builder.append(String.join(";", validationErrors));
++ final StringBuilder builder = new StringBuilder()
++ .append(numValidationErrors)
++ .append(" validation ")
++ .append(numValidationErrors > 1 ? "errors detected: " : "error detected: ")
++ .append(String.join(";", validationErrors));
+ final String validationMessage = builder.toString();
- final OSTSException exception = new OSTSException(
- VALIDATION_ERROR, validationMessage, BAD_REQUEST.getStatusCode());
++ final OSTSException exception = new OSTSException(STS_VALIDATION_ERROR).withMessage(validationMessage);
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, exception));
+ throw exception;
+ }
+
+ final String assumedRoleUserArn = S3STSUtils.toAssumedRoleUserArn(roleArn, roleSessionName);
+ try {
+ final AssumeRoleResponseInfo responseInfo = getClient()
+ .getObjectStore()
+ .assumeRole(roleArn, roleSessionName, duration, awsIamSessionPolicy, requestId);
+ // Generate AssumeRole response
+ final String responseXml = generateAssumeRoleResponse(assumedRoleUserArn, responseInfo, requestId);
+
+ getAuditLogger().logWriteSuccess(buildAuditMessageForSuccess(S3GAction.ASSUME_ROLE, auditParams));
+
+ return Response.ok(responseXml)
+ .header("Content-Type", "text/xml")
+ .build();
+ } catch (IOException e) {
+ LOG.error("Error during AssumeRole processing", e);
+
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e));
+
+ if (e instanceof OMException) {
+ final OMException omException = (OMException) e;
+ if (omException.getResult() == OMException.ResultCodes.ACCESS_DENIED ||
+ omException.getResult() == OMException.ResultCodes.PERMISSION_DENIED ||
+ omException.getResult() == OMException.ResultCodes.TOKEN_EXPIRED) {
- throw new OSTSException(
- ACCESS_DENIED, "User is not authorized to perform: sts:AssumeRole on resource: " + roleArn,
- FORBIDDEN.getStatusCode());
++ throw new OSTSException(ACCESS_DENIED)
++ .withMessage("User is not authorized to perform: sts:AssumeRole on resource: " + roleArn);
+ }
+ if (omException.getResult() == OMException.ResultCodes.INVALID_TOKEN) {
- throw new OSTSException(
- INVALID_CLIENT_TOKEN_ID, "The security token included in the request is invalid.",
- FORBIDDEN.getStatusCode());
++ throw new OSTSException(STS_INVALID_CLIENT_TOKEN_ID);
+ }
+ if (omException.getResult() == OMException.ResultCodes.NOT_SUPPORTED_OPERATION ||
+ omException.getResult() == OMException.ResultCodes.FEATURE_NOT_ENABLED) {
- throw new OSTSException(UNSUPPORTED_OPERATION, omException.getMessage(), NOT_IMPLEMENTED.getStatusCode());
++ throw new OSTSException(STS_UNSUPPORTED_OPERATION).withMessage(omException.getMessage());
+ }
+ if (omException.getResult() == OMException.ResultCodes.INVALID_REQUEST) {
- throw new OSTSException(VALIDATION_ERROR, omException.getMessage(), BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_VALIDATION_ERROR).withMessage(omException.getMessage());
+ }
+ if (omException.getResult() == OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT) {
- throw new OSTSException(MALFORMED_POLICY_DOCUMENT, omException.getMessage(), BAD_REQUEST.getStatusCode());
++ throw new OSTSException(STS_MALFORMED_POLICY_DOCUMENT).withMessage(omException.getMessage());
+ }
+ }
- throw new OSTSException(
- INTERNAL_FAILURE, "An internal error has occurred.", INTERNAL_SERVER_ERROR.getStatusCode(), "Receiver");
++ throw new OSTSException(STS_INTERNAL_FAILURE, e).withType("Receiver");
+ } catch (Exception e) {
+ getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e));
+ throw e;
+ }
+ }
+
+ private String generateAssumeRoleResponse(String assumedRoleUserArn, AssumeRoleResponseInfo responseInfo,
+ String requestId) throws IOException {
+ final String accessKeyId = responseInfo.getAccessKeyId();
+ final String secretAccessKey = responseInfo.getSecretAccessKey();
+ final String sessionToken = responseInfo.getSessionToken();
+ final String assumedRoleId = responseInfo.getAssumedRoleId();
+
+ final String expiration = DateTimeFormatter.ISO_INSTANT.format(
+ Instant.ofEpochSecond(responseInfo.getExpirationEpochSeconds()).atOffset(ZoneOffset.UTC).toInstant());
+
+ try {
+ final S3AssumeRoleResponseXml response = new S3AssumeRoleResponseXml();
+ final S3AssumeRoleResponseXml.AssumeRoleResult result = new S3AssumeRoleResponseXml.AssumeRoleResult();
+ final S3AssumeRoleResponseXml.Credentials credentials = new S3AssumeRoleResponseXml.Credentials();
+ credentials.setAccessKeyId(accessKeyId);
+ credentials.setSecretAccessKey(secretAccessKey);
+ credentials.setSessionToken(sessionToken);
+ credentials.setExpiration(expiration);
+ result.setCredentials(credentials);
+ final S3AssumeRoleResponseXml.AssumedRoleUser user = new S3AssumeRoleResponseXml.AssumedRoleUser();
+ user.setAssumedRoleId(assumedRoleId);
+ user.setArn(assumedRoleUserArn);
+ result.setAssumedRoleUser(user);
+ response.setAssumeRoleResult(result);
+ final S3AssumeRoleResponseXml.ResponseMetadata meta = new S3AssumeRoleResponseXml.ResponseMetadata();
+ meta.setRequestId(requestId);
+ response.setResponseMetadata(meta);
+
+ final Marshaller marshaller = JAXB_CONTEXT.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ final StringWriter stringWriter = new StringWriter();
+ marshaller.marshal(response, stringWriter);
+ return stringWriter.toString();
+ } catch (JAXBException e) {
+ throw new IOException("Failed to marshal AssumeRole response", e);
+ }
+ }
+}
+
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
index fd272fe02fc,2644150bd34..97355f10e81
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exceptions.java
@@@ -48,37 -44,4 +45,36 @@@ public class TestOS3Exceptions
ex.getRequestId());
assertEquals(expected, val);
}
+
+ /**
+ * AWS S3 returns HTTP 400 Bad Request for ExpiredToken (not 403).
+ */
+ @Test
+ public void testExpiredTokenUsesBadRequestHttpStatus() {
+ assertEquals(HTTP_BAD_REQUEST, S3ErrorTable.EXPIRED_TOKEN.getHttpCode());
+ final OS3Exception fromTable = S3ErrorTable.newError(S3ErrorTable.EXPIRED_TOKEN, "resource");
+ assertEquals(HTTP_BAD_REQUEST, fromTable.getHttpCode());
+ }
+
+ @Test
+ public void testOS3ExceptionWithToken0() {
- OS3Exception ex = new OS3Exception("ExpiredToken", "The provided token has expired.", 400);
- ex = S3ErrorTable.newError(ex, "resource");
++ final OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.EXPIRED_TOKEN, "resource");
+ ex.setRequestId(OzoneUtils.getRequestID());
+ ex.setHostId(OzoneUtils.getRequestID());
+ ex.setToken0("token-value");
+
+ final String val = ex.toXml();
+ final String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>%n" +
+ "<Error>%n" +
+ " <Code>%s</Code>%n" +
+ " <Message>%s</Message>%n" +
+ " <Resource>%s</Resource>%n" +
+ " <RequestId>%s</RequestId>%n" +
+ " <HostId>%s</HostId>%n" +
+ " <Token-0>%s</Token-0>%n" +
+ "</Error>%n";
+ final String expected = String.format(formatString, ex.getCode(), ex.getErrorMessage(), ex.getResource(),
+ ex.getRequestId(), ex.getHostId(), ex.getToken0());
+ assertEquals(expected, val);
+ }
}
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
index cd77979ec20,00000000000..184e96f0406
mode 100644,000000..100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
@@@ -1,102 -1,0 +1,104 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.StringReader;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+/**
+ * This class tests OSTSException class. It is named TestOSTSExceptions instead of
+ * TestOSTSException to avoid findbugs rule about classes ending in *Exception must
+ * extend Exception or Throwable.
+ */
+public class TestOSTSExceptions {
+
+ @Test
+ public void testOSTSException() throws Exception {
- final OSTSException ex = new OSTSException("ValidationError", "1 validation error detected", 400);
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_VALIDATION_ERROR)
++ .withMessage("1 validation error detected");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ // Ensure the response uses the default namespace (no prefix like "ns2:")
+ assertEquals("ErrorResponse", root.getNodeName());
+ assertEquals("https://sts.amazonaws.com/doc/2011-06-15/", root.getNamespaceURI());
+
+ assertEquals("Sender", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("ValidationError", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("1 validation error detected", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ @Test
+ public void testOSTSExceptionInvalidAction() throws Exception {
- final OSTSException ex = new OSTSException("InvalidAction", "Could not find operation", 400);
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_INVALID_ACTION)
++ .withMessage("Could not find operation");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ assertEquals("http://webservices.amazon.com/AWSFault/2005-15-09", root.getNamespaceURI());
+
+ assertEquals("Sender", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("InvalidAction", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("Could not find operation", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ @Test
+ public void testOSTSExceptionWithCustomType() throws Exception {
- final OSTSException ex = new OSTSException("InternalFailure", "An internal error has occurred.", 500, "Receiver");
++ final OSTSException ex = new OSTSException(S3ErrorTable.STS_INTERNAL_FAILURE).withType("Receiver");
+ final String requestId = OzoneUtils.getRequestID();
+ ex.setRequestId(requestId);
+ final String val = ex.toXml();
+
+ final Document doc = parseXml(val);
+ final Element root = doc.getDocumentElement();
+ assertEquals("ErrorResponse", root.getLocalName());
+ assertEquals("https://sts.amazonaws.com/doc/2011-06-15/", root.getNamespaceURI());
+
+ assertEquals("Receiver", doc.getElementsByTagName("Type").item(0).getTextContent());
+ assertEquals("InternalFailure", doc.getElementsByTagName("Code").item(0).getTextContent());
+ assertEquals("An internal error has occurred.", doc.getElementsByTagName("Message").item(0).getTextContent());
+ assertEquals(requestId, doc.getElementsByTagName("RequestId").item(0).getTextContent());
+ }
+
+ private static Document parseXml(String xml) throws Exception {
+ assertNotNull(xml);
+ final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ documentBuilderFactory.setNamespaceAware(true);
+ final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+ return documentBuilder.parse(new InputSource(new StringReader(xml)));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org