You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/04/20 06:26:59 UTC
[ozone] branch HDDS-4944 updated (9280b67553 -> 7719e23eec)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 9280b67553 HDDS-6375. [Multi-Tenant] Implement tenant request metrics (#3256)
add 000902d748 HDDS-6325. Fix interface ClientProtocol methods typo setThreadLocalS3Auth and clearThreadLocalS3Auth (#3093)
add f0f344d162 HDDS-6311. Fix number of keys displayed in Recon Overview. (#3081)
add 87ff3295a0 HDDS-6305: Add metrics - number of FSO bucket creates (#3077)
add 8c5fefe1aa HDDS-6330. Remove unnecessary duplicate semicolons (#3097)
add ff0209ddbd HDDS-6331. Remove toString in debug log parameters within SCMCommonPlacementPolicy (#3098)
add 2c561d0937 HDDS-6283. Change ContainerStateMachine ContainerOpExecutor name (#3055)
add eb0f56bdd9 HDDS-6324. Do not trigger CI by reopening PR (#3092)
add c87b059b0f HDDS-6284. Add BlockDeletingService worker size config (#3056)
add 51f47fc5a0 HDDS-6314. ConcurrentModificationException getting SCMContainerMetrics (#3101)
add 53fa44d1c9 HDDS-6290. operational-state and node-state options in datanode list CLI not working correctly (#3105)
add 91cf777b02 HDDS-6262. ozone insight log stops working after OM DBUpdates message (#3044)
add 07fa7759ce HDDS-6234. Repair containers affected by incorrect used bytes and block count. (#3042)
add 788ff0f39b HDDS-6212. SCM Container DB bootstrap on Recon startup for secure cluster (#3027)
add ad33ac6dee HDDS-6292. Ensure immutable ContainerReplica set is returned from ContainerStateManagerImpl (#3071)
add 8ad0796db0 HDDS-6329. New checkstyle: AvoidStarImport (#3096)
add 7f6e6fbec7 HDDS-6350. Rename TestStandardOutputUtil to observe naming conventions (#3114)
add 4b935c5250 HDDS-6328. Add metrics - number of FSO bucket deletes (#3107)
add e848ab7c3c HDDS-6327. Upgrade acceptance test doesn't collect logs when the test fails (#3106)
add 68c5ac5df4 HDDS-6351. [Docs] Update prefix-based FSO documentation. (#3115)
add 239d889344 HDDS-5267. Full Container Report can remove replicas added by an Incremental Report (#2963)
add f0ee98a45d HDDS-6363. Update checkstyle version to 9.3 (#3123)
add 1c7ff7e5e0 HDDS-6221. Update versions covered by cross-compatibility test (#3017)
add a1951003dd HDDS-5359. Incorrect BLOCKCOUNT and BYTESUSED in container DB. (#3034)
add 5f4c31a6e1 HDDS-6349. IncompleteReadError on get MPU key from TDE bucket (#3116)
add a53a3a33ec HDDS-6235. Empty KeyValueContainers are replicated without chunks directory. (#3052)
add 7310491af7 HDDS-5194. Provide kustomize descriptors for Ozone kubernetes examples (#2221)
add 93631a188c HDDS-6334. Remove ContainerID to Proto to ContainerID conversion in ContainerStateManagerImpl (#3110)
add 17d86f1792 HDDS-6361. Modify docs build flow to replace image tags with shortcodes. (#3122)
add da93105f4c HDDS-6369. PARTIAL_RENAME does not update OM DB (#3126)
add fbce851041 HDDS-6374. Fix incorrect queueTime metrics of ReplicationTask (#3135)
add ebbe9b5815 HDDS-6337. [FSO] Disable recursive access check flag for directories with no children. (#3134)
add 898ccbb46f HDDS-6376. Docs: Fix classpath for ofs and o3fs (#3138)
add 53d25b92ee HDDS-6307. Improve processing and memory efficiency of full container reports (#3085)
add 6e117f6488 HDDS-6395. Intermittent failure in TestReconScmHASnapshot.testScmHASnapshot (#3149)
add 2ed5994db1 HDDS-6268. Include audit log in acceptance test bundle (#3045)
add 5f029d0619 HDDS-6317: Export ReconTaskStatus as Prometheus metrics (#3088)
add ba31f8ce76 HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
add e9722b759b HDDS-6401. Fix flaky TestFilePerBlockStrategy.testWriteAndReadChunkMultipleTimes (#3152)
add d0cde3a161 HDDS-5712. make it configurable to trigger refresh datanode usage info before start a new balance iteration (#2944)
add b83c1f9587 HDDS-6244. ContainerBalancer metrics don't show updated values in JMX (#3049)
add e716c1810e HDDS-6321. Avoid refresh pipeline for key lookup in checkAcls (#3103)
add fe6f060d89 HDDS-6370. Document connecting IDE to Ozone on Kubernetes (#3132)
add 94598d1cf4 HDDS-6420. Ratis and Standalone ReplicationConfig should return a static instance (#3165)
add 5217134f9a HDDS-6360. Create stack dump on acceptance test failure (#3154)
add b439802f5d HDDS-6417 - Fix console output message when container balancer does not start (#3164)
add bf38696d06 HDDS-6423. Fix typo in TestTools.md (#3170)
add 2a1b6ba234 HDDS-6379. Not deducting the STANDALONE pipelines when counting pipelines on each datanode to check the pipeline limit (#3158)
add 58207a20df HDDS-6153. Add simple integration test to the read-replicas debug tool. (#3065)
add 1c58d79c11 HDDS-6415 - add Over-Utilized and Under-Utilized DN details in debug log (#3171)
add 5587023a23 HDDS-6357. RenameKey request has memory leak (#3121)
add 2a776a3745 HDDS-6306 - Fix picocli warnings in ozone container balancer help (#3162)
add ea06d26afd HDDS-6419. Provide better error message for malformed auth header (#3167)
add 759ac57a66 HDDS-6407. Unwrap RemoteException in getClientProtocol (#3157)
add 933a19c90a HDDS-5656. Move old objects to delete table on overwriting multipart objects (#2813)
add 1c414bf10e HDDS-6299. Remove Log4J2 from Ozone FS shaded jars (#3073)
add a720211b6b HDDS-6267. Recon is accessing mock classes. (#3084)
add 27aad3c8b2 HDDS-6391. ClientVersions and DatanodeVersions class to define an enum with version and description (#3155)
add 7cfd846027 HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3182)
add b01138c27f HDDS-6392. Introduce OzoneManagerVersion enum (#3159)
add bb212de993 HDDS-6418. Datanode usage info uses wrong version number (#3173)
add f826d124aa HDDS-6431. Fix usedBytes for FSO bucket (#3178)
add 5a1babfd45 HDDS-6439. Minor perf optimization in the version enums (#3189)
add 4454e6b368 HDDS-6095. Skip slow integration tests, allow repeating known flaky tests 5x in CI (#3148)
add 5f3f9fae17 HDDS-4171. Typo in MiniOzoneCluster: DEFAULT_PIPELIME_LIMIT (#3191)
add c2a67aca1b HDDS-6450. [FSO] Fix correctedSpace for usedBytes in commitKey (#3193)
add 74d92c86be HDDS-6213. Pluggable OzoneManager request handling hooks (#3104)
add 909b0d23f7 HDDS-6079. Replace Freon RK with OCKG/OCKV in acceptance tests (#2916)
add be2021a369 HDDS-6470. Fix TestOzoneManagerHAWithData#testOMRestart() (#3213)
add ea296fe7f8 HDDS-6468. Set correct resource type for ACL checks in BucketAcl and KeyAcl requests (#3209)
add 416439d7a2 HDDS-5867. Update quota when deleting open keys (#3206)
add 97d8ed73bc HDDS-6432. Smoketest for read-replicas tool expects exactly 3 datanodes (#3187)
add 64081985ab HDDS-6464. Turn off cross-check fail-fast for CI (#3208)
add ca24bdb911 HDDS-5228 Make OM FailOverProxyProvider work across threads (#3160)
add d97687cdbe HDDS-6087. Compute composite CRC file checksum using chunk checksums from DataNodes. (#2937)
add 4da4cc174f HDDS-6477. Intermittent failure in TestOzoneManagerHAMetadataOnly#testListVolumes (#3219)
add 61a313fd3b HDDS-6228. Update config keys for open key cleanup service (#3022)
add 042aa8b330 HDDS-3370. Cleanup old write-path of bucket in OM (#3183)
add 92995ec8ae HDDS-6261. OM crashes when trying to overwrite a key during upgrade downgrade testing (#3174)
add a8ed41ebd3 HDDS-6490. Replace deprecated --no-ansi option in acceptance tests (#3224)
add a5602a8b01 HDDS-6480. Add missing CleanupTableInfo to OM responses (#3220)
add 1baa6521d1 HDDS-6488. Intermittent failure in ozone debug read-replicas with one datanode STALE (#3223)
add 4ff630082b HDDS-6398. Add audit logs for DelegationToken (#3163)
add 2f2f52591c HDDS-6472. Errors in TestOzoneContainer integration test (#3216)
add cfb8bae20b HDDS-6393. Change the OzoneManager ServiceInfo message to carry the int version from OzoneManagerVersions. (#3211)
add 9f716b8aa7 HDDS-6482. Refactor Ratis Rpc and TLS config code. (#3222)
add 75f5501602 HDDS-6481. Add metrics for S3 gateway (#3221)
add b21a613146 HDDS-6473. Empty data in response for V0 GetSmallFile request (#3230)
add d35da4091d HDDS-6098. Add a metric to track amount of data committed. (#2913)
add e07c66ca27 HDDS-6425. OmMetadataManagerImpl#isBucketEmpty does not work on FSO buckets. (#3227)
add 605746617b HDDS-6497. S3 Gateway throws exception when checking jmx or prom pages from browser (#3228)
add ff9f523899 HDDS-6509. Checkstyle: Enable setterCanReturnItsClass in HiddenField (#3237)
add a8ef728055 HDDS-6499. Cleanup OMRequest class constructors. (#3248)
add 682be322fa HDDS-6367. ContainerBalancer shows incorrect iteration result sometimes (#3129)
add d1e39315f0 HDDS-6502. Blocks of old key versions are not deleted on key deletion (#3232)
add 0c4a01766b HDDS-6516. Shade Kotlin for Ozone Filesystem (#3246)
add 0ec6005ce7 HDDS-6429. getContainerReplicas should respect client version (#3231)
add 0bf2090078 HDDS-6505. TestContainerStateManagerIntegration consistently fails (#3234)
add adecbd1f92 HDDS-6362. Add aggregate metrics to ContainerBalancerMetrics (#3239)
add 0262c8ac3a HDDS-6466. Drop S3 Gateway bucket browser feature (#3241)
add 2eabcc5bd5 HDDS-5963. Implement ListBlock command (#3212)
add d93fbdfa64 HDDS-6397. Implement ContainerBalancer as an SCMService (#3153)
add 1d40e55e91 HDDS-6538. Update Spring to 5.2.20 to fix CVE-2022-22965 aka Spring4shell. (#3261)
add a6b09e777e HDDS-6456 Update RocksDB Version to 7.0.4 (#3199)
add c23aae0d93 HDDS-6435. Add read lock waiting and held time metrics (#3180)
add 68270e46bf HDDS-6414. Ozone doesn’t reclaim space after deletion of volume via ofs (#3175)
add 15a09aa127 HDDS-6523. Refine OMClientRequest Classes to handle bucket layout better. (#3265)
add 0f04fa7b17 HDDS-6537. Deprecate the renameKeys API (#3260)
add aa68b9abed HDDS-5119. Recon file count by size page has incorrect data when keys are deleted (#3269)
add 805b5e1d3b HDDS-6545. OzoneManagerUtils#getBucketLayout should not suppress bucket key lookup errors. (#3277)
add 76f809f3ec HDDS-6436. Add write lock waiting and held time metrics (#3271)
add 473627b8f5 HDDS-6554. Have the datanode heartbeat include queued command counts (#3275)
add e81ab0a3f7 HDDS-6564. DirectoryDeletionService should use getFileTable to handle bucket delete (#3284)
add 55d5ecbda2 HDDS-6552. Bump Jackson Databind (#3273)
add 64d67668d2 HDDS-5862. Datanode to Recon heartbeat interval independently configurable (#3245)
add de0b1f9b81 HDDS-6528. Adding Unit-Test cases for S3-Gateway Bucket-Endpoint Metrics. (#3263)
add 4c1ba2b998 HDDS-6535. Ozone TokenRenewer not found (#3259)
add 141ced1196 HDDS-6579. Add commandline argument for Container Info command. (#3306)
add 22cdd95a9e HDDS-6141. terminate om if statemachine is shut down by ratis (#2953)
add c583b3e7c6 HDDS-6162. Limit OM DoubleBuffer pending requests to avoid taking too much memory (#3024)
add d8a21a2bdb HDDS-6593. Bump node to v16.14.2 for Recon (#3316)
add 0b4af02bfb HDDS-6591. Javadoc - update StorageContinerLocationProtocol for change of qualified class name (#3314)
add 2851ea53f0 HDDS-5339. Document OM bootstrap and decommission process (#3299)
add 8f98af911c HDDS-6547. om snapshot failed file not deleted. (#3268)
add fe49f69a33 HDDS-6346. OmFailoverProxyProvider cleanup in loadConfigs for legacy service ids code (#3313)
add d5eb702c68 HDDS-6587. SCM Web UI is not displaying HEALTHY_READONLY header in node status (#3311)
add a0684e26c8 HDDS-6574. Set owner of buckets created via S3 Gateway to actual user rather than `s3g`; print LinkBucket owner field on the client (#3298)
add 3fd7dc6c77 HDDS-6525. Add audit log for S3Gateway (#3252)
add 0b6f46714f HDDS-6562. Exclude specific operations in Audit log (#3289)
new 7719e23eec Merge remote-tracking branch 'asf/master' into HDDS-4944
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 12 +-
dev-support/annotations/pom.xml | 114 ++++
.../RequestFeatureValidatorProcessor.java | 289 ++++++++++
.../org/apache/ozone/annotations/package-info.java | 5 +
.../services/javax.annotation.processing.Processor | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
hadoop-hdds/common/pom.xml | 13 +-
...DatanodeVersions.java => ComponentVersion.java} | 24 +-
.../org/apache/hadoop/hdds/DatanodeVersion.java | 71 +++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 +
.../hadoop/hdds/client/RatisReplicationConfig.java | 34 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/StandaloneReplicationConfig.java | 34 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 66 ++-
.../RequestTypeDependentRetryPolicyCreator.java | 6 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 21 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 21 +-
.../ContainerCommandResponseBuilders.java | 13 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 75 +++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 34 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 72 +++
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 49 +-
.../common/src/main/resources/ozone-default.xml | 91 ++-
.../hdds/TestComponentVersionInvariants.java | 98 ++++
.../client/TestReplicationConfigValidator.java | 16 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 11 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 9 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 21 +
.../ozone/container/ContainerTestHelper.java | 96 +---
.../TestDefaultUpgradeFinalizationExecutor.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +-
.../ozone/container/common/impl/ContainerData.java | 46 +-
.../ozone/container/common/impl/ContainerSet.java | 16 +-
.../common/interfaces/ContainerInspector.java | 72 +++
.../ozone/container/common/interfaces/Handler.java | 12 +-
.../common/report/ContainerReportPublisher.java | 5 +-
.../common/report/IncrementalReportSender.java | 30 +
.../common/statemachine/DatanodeStateMachine.java | 37 +-
.../common/statemachine/StateContext.java | 157 ++++--
.../commandhandler/CommandDispatcher.java | 16 +
.../commandhandler/CommandHandler.java | 10 +
.../CreatePipelineCommandHandler.java | 13 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 11 +
.../DeleteContainerCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../states/datanode/RunningDatanodeState.java | 16 +-
.../states/endpoint/HeartbeatEndpointTask.java | 44 +-
.../states/endpoint/RegisterEndpointTask.java | 6 +-
.../server/ratis/ContainerStateMachine.java | 7 +-
.../transport/server/ratis/XceiverServerRatis.java | 18 +-
.../common/utils/ContainerInspectorUtil.java | 87 +++
.../container/common/volume/AbstractFuture.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 76 ++-
.../container/keyvalue/KeyValueContainerData.java | 10 +-
.../KeyValueContainerMetadataInspector.java | 463 ++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 109 ++--
.../container/keyvalue/TarContainerPacker.java | 55 +-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 114 ++--
.../keyvalue/impl/ChunkManagerDispatcher.java | 18 +-
.../background/BlockDeletingService.java | 108 ++--
.../container/metadata/AbstractDatanodeStore.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +-
.../container/replication/MeasuredReplicator.java | 3 +-
.../replication/ReplicationSupervisor.java | 2 +-
.../commands/RefreshVolumeUsageCommand.java | 57 ++
.../hadoop/ozone/container/common/ScmTestMock.java | 7 +-
.../container/common/TestBlockDeletingService.java | 6 +-
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 47 --
.../container/common/impl/TestHddsDispatcher.java | 27 +-
.../common/statemachine/TestStateContext.java | 107 +++-
.../TestCreatePipelineCommandHandler.java | 36 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 26 +-
.../common/volume/TestStorageVolumeChecker.java | 12 +-
.../keyvalue/TestKeyValueBlockIterator.java | 10 +-
.../container/keyvalue/TestKeyValueContainer.java | 67 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 158 +-----
...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++----
.../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 26 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 -
.../keyvalue/impl/CommonChunkManagerTestCases.java | 1 -
.../keyvalue/impl/TestBlockManagerImpl.java | 57 +-
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../replication/TestMeasuredReplicator.java | 15 +
.../testutils/BlockDeletingServiceTestImpl.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 +
hadoop-hdds/docs/content/concept/Containers.md | 3 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 9 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 41 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 +--
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
.../docs/content/start/StartFromDockerHub.md | 6 -
.../docs/content/start/StartFromDockerHub.zh.md | 5 -
hadoop-hdds/docs/content/tools/TestTools.md | 4 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 4 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 ++
.../themes/ozonedoc/layouts/shortcodes/image.html | 2 +-
hadoop-hdds/framework/pom.xml | 8 +
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 37 ++
...lockLocationProtocolClientSideTranslatorPB.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 24 +-
.../scm/update/client/UpdateServiceConfig.java | 5 +-
.../authority/PKIProfiles/DefaultProfile.java | 32 +-
...ateClient.java => CommonCertificateClient.java} | 116 ++--
.../certificate/client/OMCertificateClient.java | 79 +--
.../certificate/client/ReconCertificateClient.java | 40 +-
...va => FixedThreadPoolWithAffinityExecutor.java} | 70 ++-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 14 +
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 75 ---
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 50 +-
.../hdds/utils/db/cache/PartialTableCache.java | 56 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../client/TestDefaultCertificateClient.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 35 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 77 ++-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
hadoop-hdds/interface-client/pom.xml | 5 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 7 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 17 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 8 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +-
.../container/AbstractContainerReportHandler.java | 44 +-
.../hdds/scm/container/ContainerManager.java | 7 +
.../hdds/scm/container/ContainerManagerImpl.java | 78 +--
.../hdds/scm/container/ContainerReportHandler.java | 141 +++--
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../scm/container/ContainerStateManagerImpl.java | 24 +-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../scm/container/balancer/ContainerBalancer.java | 526 ++++++++++++------
.../balancer/ContainerBalancerConfiguration.java | 23 +-
.../balancer/ContainerBalancerMetrics.java | 163 ++++--
.../IllegalContainerBalancerStateException.java | 36 +-
...lidContainerBalancerConfigurationException.java | 37 +-
.../scm/container/states/ContainerStateMap.java | 27 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 37 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 8 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +-
...ffer.java => SCMHADBTransactionBufferStub.java} | 8 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
...MockSCMHAManager.java => SCMHAManagerStub.java} | 42 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 14 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 6 +-
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 6 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 15 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 18 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 34 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 6 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 4 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 54 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 6 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 3 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 28 +
...inerLocationProtocolServerSideTranslatorPB.java | 26 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 2 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 2 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 54 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 16 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 62 ++-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 14 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 7 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 21 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +-
.../container/TestCloseContainerEventHandler.java | 8 +-
.../scm/container/TestContainerManagerImpl.java | 26 +-
.../scm/container/TestContainerReportHandler.java | 34 +-
.../scm/container/TestContainerStateManager.java | 14 +-
.../TestIncrementalContainerReportHandler.java | 30 +-
.../hdds/scm/container/TestReplicationManager.java | 140 ++---
.../scm/container/TestUnknownContainerReport.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 233 +++++---
.../states/TestContainerReplicaCount.java | 5 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 15 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 6 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 7 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 24 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 12 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 4 +-
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 119 ++--
.../scm/pipeline/TestPipelinePlacementPolicy.java | 158 +++++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 62 ++-
.../scm/pipeline/TestRatisPipelineProvider.java | 50 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 18 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 20 +-
.../TestOneReplicaPipelineSafeModeRule.java | 10 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 22 +-
.../scm/server/TestSCMBlockProtocolServer.java | 10 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +-
.../server/TestStorageContainerManagerStarter.java | 6 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 14 +
hadoop-hdds/test-utils/pom.xml | 5 +
.../org/apache/ozone/test/GenericTestUtils.java | 18 +-
.../java/org/apache/ozone/test/tag/Flaky.java} | 36 +-
.../main/java/org/apache/ozone/test/tag/Slow.java} | 35 +-
.../org/apache/ozone/test/tag/package-info.java} | 23 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 18 +-
.../hdds/scm/cli/ContainerOperationClient.java | 17 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 7 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 6 +-
.../datanode/TestContainerBalancerSubCommand.java | 22 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 1 +
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../client/checksum/BaseFileChecksumHelper.java | 54 +-
.../checksum/ReplicatedFileChecksumHelper.java | 44 +-
.../client/io/BlockOutputStreamEntryPool.java | 1 -
.../client/io/MultipartCryptoKeyInputStream.java | 4 +
.../ozone/client/protocol/ClientProtocol.java | 5 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 61 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 39 +-
.../hadoop/ozone/client/rpc/RpcClientTest.java | 279 +++++-----
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 21 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 180 +++---
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 3 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 4 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 25 +-
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 +++
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 +++++++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 273 ++++++++-
.../RangerRestMultiTenantAccessController.java | 357 ++++++------
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 3 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 37 +-
.../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 12 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 4 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 227 ++++++++
.../ozone/security/acl/TestOzoneObjInfo.java | 5 +-
hadoop-ozone/csi/pom.xml | 4 +
hadoop-ozone/datanode/pom.xml | 1 +
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
.../dev-support/checks/_mvn_unit_report.sh | 24 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-topology/test.sh | 6 -
.../dist/src/main/compose/ozone/docker-config | 4 +
.../dist/src/main/compose/ozone/prometheus.yml | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../src/main/compose/ozonesecure/docker-config | 4 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 86 ++-
.../dist/src/main/compose/upgrade/testlib.sh | 9 +-
.../non-rolling-upgrade/1.1.0-1.2.0/callback.sh | 6 +
.../dist/src/main/compose/xcompat/clients.yaml | 18 +
.../dist/src/main/compose/xcompat/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +-
hadoop-ozone/dist/src/main/k8s/README.md | 68 +++
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/examples/getting-started/Flekszible | 2 +
.../examples/getting-started/config-configmap.yaml | 1 +
.../examples/getting-started/kustomization.yaml} | 17 +-
.../dist/src/main/k8s/examples/minikube/Flekszible | 2 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../main/k8s/examples/minikube/kustomization.yaml} | 17 +-
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../k8s/examples/ozone-dev/kustomization.yaml} | 26 +-
.../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 +
.../main/k8s/examples/ozone-ha/kustomization.yaml} | 13 +-
.../dist/src/main/k8s/examples/ozone/Flekszible | 3 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../main/k8s/examples/ozone/kustomization.yaml} | 13 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
.../src/main/smoketest/admincli/datanode.robot | 6 +-
.../main/smoketest/auditparser/auditparser.robot | 22 +-
.../dist/src/main/smoketest/basic/basic.robot | 9 +-
.../debug/ozone-debug-corrupt-block.robot | 49 ++
.../ozone-debug-dead-datanode.robot} | 35 +-
.../debug/ozone-debug-stale-datanode.robot | 48 ++
.../main/smoketest/debug/ozone-debug-tests.robot | 51 ++
.../src/main/smoketest/debug/ozone-debug.robot | 93 +++-
.../dist/src/main/smoketest/freon/generate.robot | 19 +-
.../dist/src/main/smoketest/freon/remove.robot | 21 +-
.../dist/src/main/smoketest/freon/validate.robot | 13 +-
.../dist/src/main/smoketest/omha/om-prepare.robot | 3 +-
.../dist/src/main/smoketest/ozone-lib/freon.robot | 65 +++
.../dist/src/main/smoketest/recon/recon-api.robot | 6 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 21 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 15 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++
.../dist/src/main/smoketest/spnego/web.robot | 4 -
.../dist/src/main/smoketest/upgrade/generate.robot | 10 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 90 +++
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 25 +
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 26 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 2 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 +
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 245 ++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 8 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 48 ++
.../rooted/ITestRootedOzoneContractRootDir.java | 1 -
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 23 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 7 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 19 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 5 +-
.../TestContainerStateManagerIntegration.java | 127 ++---
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 28 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 29 +-
.../TestRatisPipelineCreateAndDestroy.java | 31 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 7 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 33 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 8 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 20 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 5 +-
...OutputUtil.java => StandardOutputTestBase.java} | 2 +-
.../hadoop/ozone/TestContainerOperations.java | 35 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 36 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 8 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +-
.../hadoop/ozone/TestStorageContainerManager.java | 5 +-
.../ozone/client/TestOzoneClientFactory.java | 75 +++
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 23 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 23 +-
.../rpc/TestCloseContainerHandlingByClient.java | 14 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../TestContainerStateMachineFailureOnRead.java | 21 +-
.../rpc/TestContainerStateMachineFailures.java | 13 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 14 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 144 ++++-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 28 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 18 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 33 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 3 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 10 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 24 +-
.../ozone/client/rpc/TestWatchForCommit.java | 12 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 2 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 8 +-
.../commandhandler/TestBlockDeletion.java | 21 +-
.../TestCloseContainerByPipeline.java | 24 +-
.../commandhandler/TestCloseContainerHandler.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
...ler.java => TestRefreshVolumeUsageHandler.java} | 121 ++--
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/ozoneimpl/TestOzoneContainer.java | 152 ++---
.../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +-
.../container/server/TestContainerServer.java | 23 +-
.../server/TestSecureContainerServer.java | 10 -
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 13 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 16 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 19 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +-
.../ozone/om/TestContainerReportWithKeys.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 39 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 -
.../apache/hadoop/ozone/om/TestObjectStore.java | 12 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 +++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 77 ++-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 7 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 5 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 25 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 2 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 40 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 62 +--
.../ozone/om/TestOzoneManagerHAWithFailover.java | 4 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 2 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 49 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 9 +-
.../om/multitenant/TestMultiTenantVolume.java | 2 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 35 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 8 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 7 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 5 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 18 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 27 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +-
hadoop-ozone/interface-client/pom.xml | 4 +
.../src/main/proto/OmClientProtocol.proto | 3 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 6 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 6 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 3 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 3 +-
hadoop-ozone/ozone-manager/pom.xml | 5 +
.../org/apache/hadoop/ozone/om/BucketManager.java | 22 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 255 ---------
.../hadoop/ozone/om/DirectoryDeletingService.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 9 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 169 +++---
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 31 ++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 173 ++++--
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 38 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 94 +++-
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 20 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 17 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 77 ++-
.../BucketLayoutAwareOMKeyRequestFactory.java | 314 +++++++++++
.../hadoop/ozone/om/request/OMClientRequest.java | 15 +-
.../ozone/om/request/OMClientRequestUtils.java | 50 ++
.../ozone/om/request/OMKeyRequestFactory.java | 139 -----
.../om/request/bucket/OMBucketCreateRequest.java | 18 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 19 -
.../file/OMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 17 -
.../ozone/om/request/file/OMFileRequest.java | 60 +-
.../om/request/key/OMAllocateBlockRequest.java | 14 -
.../ozone/om/request/key/OMKeyCommitRequest.java | 67 +--
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 18 -
.../ozone/om/request/key/OMKeyDeleteRequest.java | 15 -
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 19 -
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 137 +++--
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 48 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++++
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +-
.../S3InitiateMultipartUploadRequest.java | 15 -
.../multipart/S3MultipartUploadAbortRequest.java | 15 -
.../S3MultipartUploadCommitPartRequest.java | 15 -
.../S3MultipartUploadCompleteRequest.java | 87 ++-
.../S3MultipartUploadCompleteRequestWithFSO.java | 15 +-
.../om/request/s3/security/OMSetSecretRequest.java | 5 +-
.../security/OMCancelDelegationTokenRequest.java | 42 +-
.../security/OMGetDelegationTokenRequest.java | 54 +-
.../security/OMRenewDelegationTokenRequest.java | 51 +-
.../validation/RequestFeatureValidator.java | 99 ++++
.../request/validation/RequestProcessingPhase.java | 29 +-
.../om/request/validation/RequestValidations.java | 107 ++++
.../om/request/validation/ValidationCondition.java | 55 ++
.../om/request/validation/ValidationContext.java | 52 ++
.../om/request/validation/ValidatorRegistry.java | 201 +++++++
.../ozone/om/request/validation/package-info.java | 62 +++
.../hadoop/ozone/om/response/OMClientResponse.java | 16 -
.../om/response/bucket/OMBucketCreateResponse.java | 3 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 3 +-
.../response/file/OMFileCreateResponseWithFSO.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 3 +-
.../key/OMAllocateBlockResponseWithFSO.java | 3 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 4 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 3 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysDeleteResponse.java | 12 +-
...thFSO.java => OMKeysDeleteResponseWithFSO.java} | 76 ++-
.../om/response/key/OMKeysRenameResponse.java | 10 +
.../om/response/key/OMOpenKeysDeleteResponse.java | 18 +-
.../multipart/S3MultipartUploadAbortResponse.java | 9 +-
.../S3MultipartUploadCommitPartResponse.java | 9 +-
.../S3MultipartUploadCompleteResponse.java | 36 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 9 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 25 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 23 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 14 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 243 ++++----
.../hadoop/ozone/om/TestKeyDeletingService.java | 142 +++--
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 23 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 50 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../apache/hadoop/ozone/om/TestTrashService.java | 6 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../ozone/om/request/OMRequestTestUtils.java | 16 +
.../request/TestBucketLayoutAwareOMKeyFactory.java | 166 ++++++
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +
.../bucket/TestOMBucketDeleteRequestWithFSO.java | 76 +++
.../request/file/TestOMDirectoryCreateRequest.java | 37 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../file/TestOMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 7 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 97 +++-
.../om/request/key/TestOMKeyRenameRequest.java | 14 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 35 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++++
.../om/request/key/TestOMKeysRenameRequest.java | 4 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 178 +++---
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../TestS3MultipartUploadAbortRequest.java | 4 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 5 +
.../TestS3MultipartUploadCommitPartRequest.java | 6 +-
.../TestS3MultipartUploadCompleteRequest.java | 40 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +-
.../security/TestOMGetDelegationTokenRequest.java | 10 +-
.../upgrade/TestOMCancelPrepareRequest.java | 2 +-
.../TestRequestFeatureValidatorProcessor.java | 524 +++++++++++++++++
.../request/validation/TestRequestValidations.java | 349 ++++++++++++
.../request/validation/TestValidatorRegistry.java | 215 +++++++
.../GeneralValidatorsForTesting.java | 190 +++++++
.../ValidatorsForOnlyOldClientValidations.java | 43 ++
.../ozone/om/response/TestCleanupTableInfo.java | 6 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 61 +-
.../s3/multipart/TestS3MultipartResponse.java | 19 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 17 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 2 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++++-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 2 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 3 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
.../org.apache.hadoop.security.token.DtFetcher | 0
...rg.apache.hadoop.security.token.TokenIdentifier | 0
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 11 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 2 +-
hadoop-ozone/recon-codegen/pom.xml | 4 +
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 125 +++++
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 +
.../ozone/recon/api/ClusterStateEndpoint.java | 15 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 +++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 20 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 +-
.../scm/ReconStorageContainerManagerFacade.java | 18 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 20 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestOpenContainerCount.java | 16 +-
.../recon/fsck/TestContainerHealthStatus.java | 4 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +-
.../TestContainerHealthTaskRecordGenerator.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 20 +-
.../ozone/recon/scm/TestReconContainerManager.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 12 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 15 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 4 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 4 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 256 ++++++---
hadoop-ozone/s3gateway/pom.xml | 8 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 +++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 9 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 24 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 232 +++++---
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 69 ++-
.../ozone/s3/endpoint/ListBucketResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 161 +++++-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 46 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 368 ++++++++++++
.../hadoop/ozone/s3/metrics/package-info.java | 23 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 20 +
.../s3gateway/src/main/resources/browser.html | 617 ---------------------
.../hadoop/ozone/client/OzoneBucketStub.java | 3 +-
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 158 ++++++
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 284 ++++++++++
.../src/test/resources/auditlog.properties | 76 +++
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 3 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 5 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../freon/containergenerator/GeneratorOm.java | 5 +-
.../freon/containergenerator/GeneratorScm.java | 4 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 5 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +-
pom.xml | 92 ++-
688 files changed, 16953 insertions(+), 6627 deletions(-)
create mode 100644 dev-support/annotations/pom.xml
create mode 100644 dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
create mode 100644 dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
rename .github/workflows/cancel-ci.yaml => dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor (61%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/{DatanodeVersions.java => ComponentVersion.java} (67%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersion.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/{AuditLoggerType.java => S3GAction.java} (60%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerInspector.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/IncrementalReportSender.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerInspectorUtil.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java
copy hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/{TestKeyValueContainerCheck.java => TestKeyValueContainerIntegrityChecks.java} (52%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
create mode 100644 hadoop-hdds/docs/dev-support/bin/make_images_responsive.py
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/{OMCertificateClient.java => CommonCertificateClient.java} (51%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java (52%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{FixedThreadPoolExecutor.java => FixedThreadPoolWithAffinityExecutor.java} (59%)
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/EpochEntry.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/IllegalContainerBalancerStateException.java (52%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/InvalidContainerBalancerConfigurationException.java (50%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHADBTransactionBuffer.java => SCMHADBTransactionBufferStub.java} (92%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHAManager.java => SCMHAManagerStub.java} (89%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java => test-utils/src/main/java/org/apache/ozone/test/tag/Flaky.java} (56%)
rename hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java => test-utils/src/main/java/org/apache/ozone/test/tag/Slow.java} (59%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java => test-utils/src/main/java/org/apache/ozone/test/tag/package-info.java} (62%)
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/LockUsageInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockMetrics.java
create mode 100644 hadoop-ozone/dist/src/main/k8s/README.md
copy hadoop-ozone/{ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer => dist/src/main/k8s/examples/getting-started/kustomization.yaml} (71%)
copy hadoop-ozone/{ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer => dist/src/main/k8s/examples/minikube/kustomization.yaml} (71%)
copy hadoop-ozone/{ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer => dist/src/main/k8s/examples/ozone-dev/kustomization.yaml} (56%)
copy hadoop-ozone/{ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer => dist/src/main/k8s/examples/ozone-ha/kustomization.yaml} (78%)
copy hadoop-ozone/{ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer => dist/src/main/k8s/examples/ozone/kustomization.yaml} (78%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
copy hadoop-ozone/dist/src/main/smoketest/{freon/validate.robot => debug/ozone-debug-dead-datanode.robot} (50%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
create mode 100644 hadoop-ozone/dist/src/shell/conf/s3g-audit-log4j2.properties
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedDDSWithFSO.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestStandardOutputUtil.java => StandardOutputTestBase.java} (98%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientFactory.java
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{TestCloseContainerHandler.java => TestRefreshVolumeUsageHandler.java} (55%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/BucketLayoutAwareOMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequestUtils.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java (69%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/{OMKeyDeleteResponseWithFSO.java => OMKeysDeleteResponseWithFSO.java} (55%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestBucketLayoutAwareOMKeyFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (92%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconTaskStatusMetrics.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java => hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/package-info.java (62%)
delete mode 100644 hadoop-ozone/s3gateway/src/main/resources/browser.html
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
create mode 100644 hadoop-ozone/s3gateway/src/test/resources/auditlog.properties
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into HDDS-4944
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7719e23eec4c4122d0fe7d28ca3965ae198c54b8
Merge: 9280b67553 0b6f46714f
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Tue Apr 19 12:08:26 2022 -0700
Merge remote-tracking branch 'asf/master' into HDDS-4944
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
Also modified:
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerRestMultiTenantAccessController.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/OMSetSecretRequest.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
.github/workflows/post-commit.yml | 12 +-
dev-support/annotations/pom.xml | 114 ++++
.../RequestFeatureValidatorProcessor.java | 289 ++++++++++
.../org/apache/ozone/annotations/package-info.java | 5 +
.../services/javax.annotation.processing.Processor | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
hadoop-hdds/common/pom.xml | 13 +-
...DatanodeVersions.java => ComponentVersion.java} | 24 +-
.../org/apache/hadoop/hdds/DatanodeVersion.java | 71 +++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 +
.../hadoop/hdds/client/RatisReplicationConfig.java | 34 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/StandaloneReplicationConfig.java | 34 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 66 ++-
.../RequestTypeDependentRetryPolicyCreator.java | 6 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 21 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 21 +-
.../ContainerCommandResponseBuilders.java | 13 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 75 +++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 34 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 72 +++
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 49 +-
.../common/src/main/resources/ozone-default.xml | 91 ++-
.../hdds/TestComponentVersionInvariants.java | 98 ++++
.../client/TestReplicationConfigValidator.java | 16 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 11 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 9 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 21 +
.../ozone/container/ContainerTestHelper.java | 96 +---
.../TestDefaultUpgradeFinalizationExecutor.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +-
.../ozone/container/common/impl/ContainerData.java | 46 +-
.../ozone/container/common/impl/ContainerSet.java | 16 +-
.../common/interfaces/ContainerInspector.java | 72 +++
.../ozone/container/common/interfaces/Handler.java | 12 +-
.../common/report/ContainerReportPublisher.java | 5 +-
.../common/report/IncrementalReportSender.java | 30 +
.../common/statemachine/DatanodeStateMachine.java | 37 +-
.../common/statemachine/StateContext.java | 157 ++++--
.../commandhandler/CommandDispatcher.java | 16 +
.../commandhandler/CommandHandler.java | 10 +
.../CreatePipelineCommandHandler.java | 13 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 11 +
.../DeleteContainerCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../states/datanode/RunningDatanodeState.java | 16 +-
.../states/endpoint/HeartbeatEndpointTask.java | 44 +-
.../states/endpoint/RegisterEndpointTask.java | 6 +-
.../server/ratis/ContainerStateMachine.java | 7 +-
.../transport/server/ratis/XceiverServerRatis.java | 18 +-
.../common/utils/ContainerInspectorUtil.java | 87 +++
.../container/common/volume/AbstractFuture.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 76 ++-
.../container/keyvalue/KeyValueContainerData.java | 10 +-
.../KeyValueContainerMetadataInspector.java | 463 ++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 109 ++--
.../container/keyvalue/TarContainerPacker.java | 55 +-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 114 ++--
.../keyvalue/impl/ChunkManagerDispatcher.java | 18 +-
.../background/BlockDeletingService.java | 108 ++--
.../container/metadata/AbstractDatanodeStore.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +-
.../container/replication/MeasuredReplicator.java | 3 +-
.../replication/ReplicationSupervisor.java | 2 +-
.../commands/RefreshVolumeUsageCommand.java | 57 ++
.../hadoop/ozone/container/common/ScmTestMock.java | 7 +-
.../container/common/TestBlockDeletingService.java | 6 +-
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 47 --
.../container/common/impl/TestHddsDispatcher.java | 27 +-
.../common/statemachine/TestStateContext.java | 107 +++-
.../TestCreatePipelineCommandHandler.java | 36 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 26 +-
.../common/volume/TestStorageVolumeChecker.java | 12 +-
.../keyvalue/TestKeyValueBlockIterator.java | 10 +-
.../container/keyvalue/TestKeyValueContainer.java | 67 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 158 +-----
...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++----
.../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 26 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 -
.../keyvalue/impl/CommonChunkManagerTestCases.java | 1 -
.../keyvalue/impl/TestBlockManagerImpl.java | 57 +-
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../replication/TestMeasuredReplicator.java | 15 +
.../testutils/BlockDeletingServiceTestImpl.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 +
hadoop-hdds/docs/content/concept/Containers.md | 3 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 9 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 41 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 +--
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
.../docs/content/start/StartFromDockerHub.md | 6 -
.../docs/content/start/StartFromDockerHub.zh.md | 5 -
hadoop-hdds/docs/content/tools/TestTools.md | 4 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 4 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 ++
.../themes/ozonedoc/layouts/shortcodes/image.html | 2 +-
hadoop-hdds/framework/pom.xml | 8 +
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 37 ++
...lockLocationProtocolClientSideTranslatorPB.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 24 +-
.../scm/update/client/UpdateServiceConfig.java | 5 +-
.../authority/PKIProfiles/DefaultProfile.java | 32 +-
...ateClient.java => CommonCertificateClient.java} | 116 ++--
.../certificate/client/OMCertificateClient.java | 79 +--
.../certificate/client/ReconCertificateClient.java | 40 +-
...va => FixedThreadPoolWithAffinityExecutor.java} | 70 ++-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 14 +
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 75 ---
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 50 +-
.../hdds/utils/db/cache/PartialTableCache.java | 56 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../client/TestDefaultCertificateClient.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 35 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 77 ++-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
hadoop-hdds/interface-client/pom.xml | 5 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 7 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 17 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 8 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +-
.../container/AbstractContainerReportHandler.java | 44 +-
.../hdds/scm/container/ContainerManager.java | 7 +
.../hdds/scm/container/ContainerManagerImpl.java | 78 +--
.../hdds/scm/container/ContainerReportHandler.java | 141 +++--
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../scm/container/ContainerStateManagerImpl.java | 24 +-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../scm/container/balancer/ContainerBalancer.java | 526 ++++++++++++------
.../balancer/ContainerBalancerConfiguration.java | 23 +-
.../balancer/ContainerBalancerMetrics.java | 163 ++++--
.../IllegalContainerBalancerStateException.java | 36 +-
...lidContainerBalancerConfigurationException.java | 37 +-
.../scm/container/states/ContainerStateMap.java | 27 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 37 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 8 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +-
...ffer.java => SCMHADBTransactionBufferStub.java} | 8 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
...MockSCMHAManager.java => SCMHAManagerStub.java} | 42 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 14 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 6 +-
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 6 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 15 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 18 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 34 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 6 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 4 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 54 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 6 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 3 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 28 +
...inerLocationProtocolServerSideTranslatorPB.java | 26 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 2 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 2 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 54 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 16 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 62 ++-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 14 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 7 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 21 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +-
.../container/TestCloseContainerEventHandler.java | 8 +-
.../scm/container/TestContainerManagerImpl.java | 26 +-
.../scm/container/TestContainerReportHandler.java | 34 +-
.../scm/container/TestContainerStateManager.java | 14 +-
.../TestIncrementalContainerReportHandler.java | 30 +-
.../hdds/scm/container/TestReplicationManager.java | 140 ++---
.../scm/container/TestUnknownContainerReport.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 233 +++++---
.../states/TestContainerReplicaCount.java | 5 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 15 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 6 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 7 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 24 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 12 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 4 +-
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 119 ++--
.../scm/pipeline/TestPipelinePlacementPolicy.java | 158 +++++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 62 ++-
.../scm/pipeline/TestRatisPipelineProvider.java | 50 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 18 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 20 +-
.../TestOneReplicaPipelineSafeModeRule.java | 10 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 22 +-
.../scm/server/TestSCMBlockProtocolServer.java | 10 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +-
.../server/TestStorageContainerManagerStarter.java | 6 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 14 +
hadoop-hdds/test-utils/pom.xml | 5 +
.../org/apache/ozone/test/GenericTestUtils.java | 18 +-
.../java/org/apache/ozone/test/tag/Flaky.java} | 36 +-
.../main/java/org/apache/ozone/test/tag/Slow.java} | 35 +-
.../org/apache/ozone/test/tag/package-info.java} | 23 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 18 +-
.../hdds/scm/cli/ContainerOperationClient.java | 17 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 7 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 6 +-
.../datanode/TestContainerBalancerSubCommand.java | 22 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 1 +
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../client/checksum/BaseFileChecksumHelper.java | 54 +-
.../checksum/ReplicatedFileChecksumHelper.java | 44 +-
.../client/io/BlockOutputStreamEntryPool.java | 1 -
.../client/io/MultipartCryptoKeyInputStream.java | 4 +
.../ozone/client/protocol/ClientProtocol.java | 5 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 61 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 39 +-
.../hadoop/ozone/client/rpc/RpcClientTest.java | 279 +++++-----
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 21 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 180 +++---
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 3 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 4 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 25 +-
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 +++
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 +++++++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 273 ++++++++-
.../RangerRestMultiTenantAccessController.java | 357 ++++++------
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 3 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 37 +-
.../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 12 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 4 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 227 ++++++++
.../ozone/security/acl/TestOzoneObjInfo.java | 5 +-
hadoop-ozone/csi/pom.xml | 4 +
hadoop-ozone/datanode/pom.xml | 1 +
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
.../dev-support/checks/_mvn_unit_report.sh | 24 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-topology/test.sh | 6 -
.../dist/src/main/compose/ozone/docker-config | 4 +
.../dist/src/main/compose/ozone/prometheus.yml | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../src/main/compose/ozonesecure/docker-config | 4 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 86 ++-
.../dist/src/main/compose/upgrade/testlib.sh | 9 +-
.../non-rolling-upgrade/1.1.0-1.2.0/callback.sh | 6 +
.../dist/src/main/compose/xcompat/clients.yaml | 18 +
.../dist/src/main/compose/xcompat/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +-
hadoop-ozone/dist/src/main/k8s/README.md | 68 +++
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/examples/getting-started/Flekszible | 2 +
.../examples/getting-started/config-configmap.yaml | 1 +
.../examples/getting-started/kustomization.yaml} | 17 +-
.../dist/src/main/k8s/examples/minikube/Flekszible | 2 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../main/k8s/examples/minikube/kustomization.yaml} | 17 +-
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../k8s/examples/ozone-dev/kustomization.yaml} | 26 +-
.../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 +
.../main/k8s/examples/ozone-ha/kustomization.yaml} | 13 +-
.../dist/src/main/k8s/examples/ozone/Flekszible | 3 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../main/k8s/examples/ozone/kustomization.yaml} | 13 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
.../src/main/smoketest/admincli/datanode.robot | 6 +-
.../main/smoketest/auditparser/auditparser.robot | 22 +-
.../dist/src/main/smoketest/basic/basic.robot | 9 +-
.../debug/ozone-debug-corrupt-block.robot | 49 ++
.../ozone-debug-dead-datanode.robot} | 35 +-
.../debug/ozone-debug-stale-datanode.robot | 48 ++
.../main/smoketest/debug/ozone-debug-tests.robot | 51 ++
.../src/main/smoketest/debug/ozone-debug.robot | 93 +++-
.../dist/src/main/smoketest/freon/generate.robot | 19 +-
.../dist/src/main/smoketest/freon/remove.robot | 21 +-
.../dist/src/main/smoketest/freon/validate.robot | 13 +-
.../dist/src/main/smoketest/omha/om-prepare.robot | 3 +-
.../dist/src/main/smoketest/ozone-lib/freon.robot | 65 +++
.../dist/src/main/smoketest/recon/recon-api.robot | 6 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 21 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 15 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++
.../dist/src/main/smoketest/spnego/web.robot | 4 -
.../dist/src/main/smoketest/upgrade/generate.robot | 10 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 90 +++
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 25 +
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 26 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 2 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 +
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 245 ++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 8 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 48 ++
.../rooted/ITestRootedOzoneContractRootDir.java | 1 -
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 23 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 7 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 19 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 5 +-
.../TestContainerStateManagerIntegration.java | 127 ++---
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 28 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 29 +-
.../TestRatisPipelineCreateAndDestroy.java | 31 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 7 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 33 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 8 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 20 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 5 +-
...OutputUtil.java => StandardOutputTestBase.java} | 2 +-
.../hadoop/ozone/TestContainerOperations.java | 35 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 36 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 8 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +-
.../hadoop/ozone/TestStorageContainerManager.java | 5 +-
.../ozone/client/TestOzoneClientFactory.java | 75 +++
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 23 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 23 +-
.../rpc/TestCloseContainerHandlingByClient.java | 14 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../TestContainerStateMachineFailureOnRead.java | 21 +-
.../rpc/TestContainerStateMachineFailures.java | 13 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 14 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 144 ++++-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 28 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 18 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 33 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 3 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 10 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 24 +-
.../ozone/client/rpc/TestWatchForCommit.java | 12 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 2 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 8 +-
.../commandhandler/TestBlockDeletion.java | 21 +-
.../TestCloseContainerByPipeline.java | 24 +-
.../commandhandler/TestCloseContainerHandler.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
...ler.java => TestRefreshVolumeUsageHandler.java} | 121 ++--
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/ozoneimpl/TestOzoneContainer.java | 152 ++---
.../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +-
.../container/server/TestContainerServer.java | 23 +-
.../server/TestSecureContainerServer.java | 10 -
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 13 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 16 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 19 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +-
.../ozone/om/TestContainerReportWithKeys.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 39 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 -
.../apache/hadoop/ozone/om/TestObjectStore.java | 12 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 +++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 77 ++-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 7 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 5 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 25 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 2 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 40 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 62 +--
.../ozone/om/TestOzoneManagerHAWithFailover.java | 4 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 2 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 49 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 9 +-
.../om/multitenant/TestMultiTenantVolume.java | 2 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 35 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 8 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 7 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 5 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 18 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 27 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +-
hadoop-ozone/interface-client/pom.xml | 4 +
.../src/main/proto/OmClientProtocol.proto | 3 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 6 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 6 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 3 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 3 +-
hadoop-ozone/ozone-manager/pom.xml | 5 +
.../org/apache/hadoop/ozone/om/BucketManager.java | 22 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 255 ---------
.../hadoop/ozone/om/DirectoryDeletingService.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 9 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 169 +++---
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 31 ++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 173 ++++--
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 38 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 94 +++-
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 20 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 17 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 77 ++-
.../BucketLayoutAwareOMKeyRequestFactory.java | 314 +++++++++++
.../hadoop/ozone/om/request/OMClientRequest.java | 15 +-
.../ozone/om/request/OMClientRequestUtils.java | 50 ++
.../ozone/om/request/OMKeyRequestFactory.java | 139 -----
.../om/request/bucket/OMBucketCreateRequest.java | 18 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 19 -
.../file/OMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 17 -
.../ozone/om/request/file/OMFileRequest.java | 60 +-
.../om/request/key/OMAllocateBlockRequest.java | 14 -
.../ozone/om/request/key/OMKeyCommitRequest.java | 67 +--
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 18 -
.../ozone/om/request/key/OMKeyDeleteRequest.java | 15 -
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 19 -
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 137 +++--
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 48 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++++
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +-
.../S3InitiateMultipartUploadRequest.java | 15 -
.../multipart/S3MultipartUploadAbortRequest.java | 15 -
.../S3MultipartUploadCommitPartRequest.java | 15 -
.../S3MultipartUploadCompleteRequest.java | 87 ++-
.../S3MultipartUploadCompleteRequestWithFSO.java | 15 +-
.../om/request/s3/security/OMSetSecretRequest.java | 5 +-
.../security/OMCancelDelegationTokenRequest.java | 42 +-
.../security/OMGetDelegationTokenRequest.java | 54 +-
.../security/OMRenewDelegationTokenRequest.java | 51 +-
.../validation/RequestFeatureValidator.java | 99 ++++
.../request/validation/RequestProcessingPhase.java | 29 +-
.../om/request/validation/RequestValidations.java | 107 ++++
.../om/request/validation/ValidationCondition.java | 55 ++
.../om/request/validation/ValidationContext.java | 52 ++
.../om/request/validation/ValidatorRegistry.java | 201 +++++++
.../ozone/om/request/validation/package-info.java | 62 +++
.../hadoop/ozone/om/response/OMClientResponse.java | 16 -
.../om/response/bucket/OMBucketCreateResponse.java | 3 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 3 +-
.../response/file/OMFileCreateResponseWithFSO.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 3 +-
.../key/OMAllocateBlockResponseWithFSO.java | 3 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 4 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 3 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysDeleteResponse.java | 12 +-
...thFSO.java => OMKeysDeleteResponseWithFSO.java} | 76 ++-
.../om/response/key/OMKeysRenameResponse.java | 10 +
.../om/response/key/OMOpenKeysDeleteResponse.java | 18 +-
.../multipart/S3MultipartUploadAbortResponse.java | 9 +-
.../S3MultipartUploadCommitPartResponse.java | 9 +-
.../S3MultipartUploadCompleteResponse.java | 36 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 9 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 25 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 23 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 14 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 243 ++++----
.../hadoop/ozone/om/TestKeyDeletingService.java | 142 +++--
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 23 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 50 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../apache/hadoop/ozone/om/TestTrashService.java | 6 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../ozone/om/request/OMRequestTestUtils.java | 16 +
.../request/TestBucketLayoutAwareOMKeyFactory.java | 166 ++++++
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +
.../bucket/TestOMBucketDeleteRequestWithFSO.java | 76 +++
.../request/file/TestOMDirectoryCreateRequest.java | 37 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../file/TestOMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 7 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 97 +++-
.../om/request/key/TestOMKeyRenameRequest.java | 14 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 35 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++++
.../om/request/key/TestOMKeysRenameRequest.java | 4 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 178 +++---
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../TestS3MultipartUploadAbortRequest.java | 4 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 5 +
.../TestS3MultipartUploadCommitPartRequest.java | 6 +-
.../TestS3MultipartUploadCompleteRequest.java | 40 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +-
.../security/TestOMGetDelegationTokenRequest.java | 10 +-
.../upgrade/TestOMCancelPrepareRequest.java | 2 +-
.../TestRequestFeatureValidatorProcessor.java | 524 +++++++++++++++++
.../request/validation/TestRequestValidations.java | 349 ++++++++++++
.../request/validation/TestValidatorRegistry.java | 215 +++++++
.../GeneralValidatorsForTesting.java | 190 +++++++
.../ValidatorsForOnlyOldClientValidations.java | 43 ++
.../ozone/om/response/TestCleanupTableInfo.java | 6 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 61 +-
.../s3/multipart/TestS3MultipartResponse.java | 19 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 17 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 2 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++++-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 2 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 3 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
.../org.apache.hadoop.security.token.DtFetcher} | 2 +-
...g.apache.hadoop.security.token.TokenIdentifier} | 6 +-
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 11 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 2 +-
hadoop-ozone/recon-codegen/pom.xml | 4 +
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 125 +++++
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 +
.../ozone/recon/api/ClusterStateEndpoint.java | 15 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 +++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 20 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 +-
.../scm/ReconStorageContainerManagerFacade.java | 18 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 20 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestOpenContainerCount.java | 16 +-
.../recon/fsck/TestContainerHealthStatus.java | 4 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +-
.../TestContainerHealthTaskRecordGenerator.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 20 +-
.../ozone/recon/scm/TestReconContainerManager.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 12 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 15 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 4 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 4 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 256 ++++++---
hadoop-ozone/s3gateway/pom.xml | 8 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 +++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 9 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 24 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 232 +++++---
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 69 ++-
.../ozone/s3/endpoint/ListBucketResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 161 +++++-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 46 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 368 ++++++++++++
.../hadoop/ozone/s3/metrics/package-info.java | 23 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 20 +
.../s3gateway/src/main/resources/browser.html | 617 ---------------------
.../hadoop/ozone/client/OzoneBucketStub.java | 3 +-
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 158 ++++++
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 284 ++++++++++
.../src/test/resources/auditlog.properties | 76 +++
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 3 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 5 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../freon/containergenerator/GeneratorOm.java | 5 +-
.../freon/containergenerator/GeneratorScm.java | 4 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 5 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +-
pom.xml | 92 ++-
688 files changed, 16956 insertions(+), 6632 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index dca3efa82b,043028347c..19d508945b
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -466,38 -450,7 +466,42 @@@ public final class OzoneConsts
public static final String OZONE_HTTP_FILTER_INITIALIZERS_SECURE =
"org.apache.hadoop.security.AuthenticationFilterInitializer";
+ public static final String DELEGATION_TOKEN_KIND = "kind";
+ public static final String DELEGATION_TOKEN_SERVICE = "service";
+ public static final String DELEGATION_TOKEN_RENEWER = "renewer";
++
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users";
+
+ // Ideally we should use /addUsersAndGroups endpoint for add user to role,
+ // but it always return 405 somehow.
+ // https://ranger.apache.org/apidocs/resource_RoleREST.html
+ // #resource_RoleREST_addUsersAndGroups_PUT
+ public static final String OZONE_OM_RANGER_ADMIN_ROLE_ADD_USER_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_USER_HTTP_ENDPOINT =
+ "/service/xusers/users/?name=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/name/";
+
+ // TODO: Use delete role endpoint
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_GROUP_HTTP_ENDPOINT =
+ "/service/xusers/secure/groups/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyName=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_POLICY_HTTP_ENDPOINT =
+ "/service/plugins/policies/";
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 59e07bc5c9,2412b889fa..a38c80e153
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@@ -160,29 -148,8 +160,29 @@@ public class ObjectStore
* @throws IOException
*/
public OzoneVolume getVolume(String volumeName) throws IOException {
- OzoneVolume volume = proxy.getVolumeDetails(volumeName);
- return volume;
+ return proxy.getVolumeDetails(volumeName);
+ }
+
+ public OzoneVolume getS3Volume() throws IOException {
+ final S3VolumeContext resp = proxy.getS3VolumeContext();
+
+ S3Auth s3Auth = proxy.getThreadLocalS3Auth();
+ // Update user principal if needed to be used for KMS client
+ if (s3Auth != null) {
+ // Update userPrincipal field with the value returned from OM. So that
+ // in multi-tenancy, KMS client can use the correct identity
+ // (instead of using accessId) to communicate with KMS.
+ LOG.debug("Updating S3Auth.userPrincipal to {}", resp.getUserPrincipal());
+ s3Auth.setUserPrincipal(resp.getUserPrincipal());
- proxy.setTheadLocalS3Auth(s3Auth);
++ proxy.setThreadLocalS3Auth(s3Auth);
+ }
+
+ OmVolumeArgs volume = resp.getOmVolumeArgs();
+ return proxy.buildOzoneVolume(volume);
+ }
+
+ public S3VolumeContext getS3VolumeContext() throws IOException {
+ return proxy.getS3VolumeContext();
}
public S3SecretValue getS3Secret(String kerberosID) throws IOException {
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 41f598d842,c38dbc753f..13afb39743
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -292,30 -307,9 +307,34 @@@ public final class OMConfigKeys
public static final long OZONE_OM_ADMIN_PROTOCOL_WAIT_BETWEEN_RETRIES_DEFAULT
= 1000;
+ public static final String OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT =
+ "ozone.om.unflushed.transaction.max.count";
+ public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
+ = 10000;
+
+ /**
+ * Temporary configuration properties for Ranger REST use in multitenancy.
+ */
+ public static final String OZONE_RANGER_OM_IGNORE_SERVER_CERT =
+ "ozone.om.ranger.ignore.cert";
+ public static final boolean OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT =
+ true;
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT =
+ "ozone.om.ranger.connection.timeout";
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT =
+ "ozone.om.ranger.connection.request.timeout";
+ public static final String
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_USER =
+ "ozone.om.ranger.https.admin.api.user";
+ // TODO: Note this should be removed once Ranger Java Client is in place.
+ // And Ranger SPNEGO auth (ranger.spnego.kerberos.principal ?) should be used
+ // instead. Or keep this solely for dev testing. See HDDS-5836.
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD =
+ "ozone.om.ranger.https.admin.api.passwd";
+ public static final String OZONE_RANGER_HTTPS_ADDRESS_KEY =
+ "ozone.om.ranger.https-address";
+ public static final String OZONE_RANGER_SERVICE =
+ "ozone.om.ranger.service";
-
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerRestMultiTenantAccessController.java
index 8931c625f6,0000000000..7184ea301d
mode 100644,000000..100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerRestMultiTenantAccessController.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerRestMultiTenantAccessController.java
@@@ -1,631 -1,0 +1,636 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.multitenant;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.http.auth.BasicUserPrincipal;
+import org.apache.kerby.util.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.lang.reflect.Type;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
- import static org.apache.hadoop.ozone.om.OMConfigKeys.*;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_HTTPS_ADDRESS_KEY;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT;
++import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_RANGER_SERVICE;
+
+/**
+ * Access controller for multi-tenancy implemented using Ranger's REST API.
+ * This class is for testing and is not intended for production use.
+ */
+public class RangerRestMultiTenantAccessController
+ implements MultiTenantAccessController {
+
+ public static final String OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/";
+ public static final String OZONE_OM_RANGER_ADMIN_ROLE_HTTP_ENDPOINT =
+ "/service/public/v2/api/roles/";
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(RangerRestMultiTenantAccessController.class);
+
+ private final OzoneConfiguration conf;
+ private boolean ignoreServerCert = false;
+ private int connectionTimeout;
+ private int connectionRequestTimeout;
+ private String authHeaderValue;
+ private final String rangerHttpsAddress;
+ private final Gson jsonConverter;
+ private final String rangerService;
+ private final Map<IAccessAuthorizer.ACLType, String> aclToString;
+ private final Map<String, IAccessAuthorizer.ACLType> stringToAcl;
+
+ public RangerRestMultiTenantAccessController(Configuration configuration)
+ throws IOException {
+ conf = new OzoneConfiguration(configuration);
+ rangerHttpsAddress = conf.get(OZONE_RANGER_HTTPS_ADDRESS_KEY);
+ rangerService = conf.get(OZONE_RANGER_SERVICE);
+
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(Policy.class, policySerializer);
+ gsonBuilder.registerTypeAdapter(Policy.class, policyDeserializer);
+ gsonBuilder.registerTypeAdapter(Role.class, roleSerializer);
+ gsonBuilder.registerTypeAdapter(Role.class, roleDeserializer);
+ gsonBuilder.registerTypeAdapter(BasicUserPrincipal.class, userSerializer);
+ jsonConverter = gsonBuilder.create();
+
+ aclToString = new EnumMap<>(IAccessAuthorizer.ACLType.class);
+ stringToAcl = new HashMap<>();
+ fillRangerAclStrings();
+ initializeRangerConnection();
+ }
+
+ private void fillRangerAclStrings() {
+ aclToString.put(IAccessAuthorizer.ACLType.ALL, "all");
+ aclToString.put(IAccessAuthorizer.ACLType.LIST, "list");
+ aclToString.put(IAccessAuthorizer.ACLType.READ, "read");
+ aclToString.put(IAccessAuthorizer.ACLType.WRITE, "write");
+ aclToString.put(IAccessAuthorizer.ACLType.CREATE, "create");
+ aclToString.put(IAccessAuthorizer.ACLType.DELETE, "delete");
+ aclToString.put(IAccessAuthorizer.ACLType.READ_ACL, "read_acl");
+ aclToString.put(IAccessAuthorizer.ACLType.WRITE_ACL, "write_acl");
+ aclToString.put(IAccessAuthorizer.ACLType.NONE, "");
+
+ stringToAcl.put("all", IAccessAuthorizer.ACLType.ALL);
+ stringToAcl.put("list", IAccessAuthorizer.ACLType.LIST);
+ stringToAcl.put("read", IAccessAuthorizer.ACLType.READ);
+ stringToAcl.put("write", IAccessAuthorizer.ACLType.WRITE);
+ stringToAcl.put("create", IAccessAuthorizer.ACLType.CREATE);
+ stringToAcl.put("delete", IAccessAuthorizer.ACLType.DELETE);
+ stringToAcl.put("read_acl", IAccessAuthorizer.ACLType.READ_ACL);
+ stringToAcl.put("write_acl", IAccessAuthorizer.ACLType.WRITE_ACL);
+ stringToAcl.put("", IAccessAuthorizer.ACLType.NONE);
+ }
+
+ private void initializeRangerConnection() {
+ setupRangerConnectionConfig();
+ if (ignoreServerCert) {
+ setupRangerIgnoreServerCertificate();
+ }
+ setupRangerConnectionAuthHeader();
+ }
+
+ private void setupRangerConnectionConfig() {
+ connectionTimeout = (int) conf.getTimeDuration(
+ OZONE_RANGER_OM_CONNECTION_TIMEOUT,
+ conf.get(
+ OZONE_RANGER_OM_CONNECTION_TIMEOUT,
+ OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT),
+ TimeUnit.MILLISECONDS);
+ connectionRequestTimeout = (int)conf.getTimeDuration(
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT,
+ conf.get(
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT,
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT),
+ TimeUnit.MILLISECONDS
+ );
+ ignoreServerCert = conf.getBoolean(
+ OZONE_RANGER_OM_IGNORE_SERVER_CERT,
+ OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT);
+ }
+
+ private void setupRangerIgnoreServerCertificate() {
+ // Create a trust manager that does not validate certificate chains
+ TrustManager[] trustAllCerts = new TrustManager[]{
+ new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ }
+ };
+
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ LOG.info("Setting DefaultSSLSocketFactory failed.");
+ }
+ }
+
+ private void setupRangerConnectionAuthHeader() {
+ String userName = conf.get(OZONE_OM_RANGER_HTTPS_ADMIN_API_USER);
+ String passwd = conf.get(OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD);
+ String auth = userName + ":" + passwd;
+ byte[] encodedAuth =
+ Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
+ authHeaderValue = "Basic " +
+ new String(encodedAuth, StandardCharsets.UTF_8);
+ }
+
+
+ @Override
+ public long createPolicy(Policy policy) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT;
+ HttpsURLConnection conn = makeHttpsPostCall(rangerAdminUrl,
+ jsonConverter.toJsonTree(policy).getAsJsonObject());
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to create policy %s. " +
+ "Http response code: %d", policy.getName(), conn.getResponseCode()));
+ }
+ String policyInfo = getResponseData(conn);
+ long policyID;
+ JsonObject jObject = new JsonParser().parse(policyInfo).getAsJsonObject();
+ policyID = jObject.get("id").getAsLong();
+ return policyID;
+ }
+
+ @Override
+ public void deletePolicy(long policyID) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT
+ + policyID;
+ HttpsURLConnection conn = makeHttpsDeleteCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to delete policy %d. " +
+ "Http response code: %d", policyID, conn.getResponseCode()));
+ }
+ }
+
+ @Override
+ public Map<Long, Policy> getPolicies() throws Exception {
+ // This API gets all policies for all services. The
+ // /public/v2/api/policies/{serviceDefName}/for-resource endpoint is
+ // supposed to get policies for only a specified service, but it does not
+ // seem to work. This implementation should be ok for testing purposes as
+ // this class is intended.
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT;
+ HttpsURLConnection conn = makeHttpsGetCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to get all policies. " +
+ "Http response code: %d", conn.getResponseCode()));
+ }
+ String allPoliciesString = getResponseData(conn);
+ // Filter out policies not for Ozone service.
+ JsonArray jsonPoliciesArray = new JsonParser().parse(allPoliciesString)
+ .getAsJsonArray();
+ Map<Long, Policy> policies = new HashMap<>();
+ for (JsonElement jsonPolicy: jsonPoliciesArray) {
+ JsonObject jsonPolicyObject = jsonPolicy.getAsJsonObject();
+ String service = jsonPolicyObject.get("service").getAsString();
+ if (service.equals(rangerService)) {
+ long id = jsonPolicyObject.get("id").getAsLong();
+ policies.put(id, jsonConverter.fromJson(jsonPolicyObject,
+ Policy.class));
+ }
+ }
+
+ return policies;
+ }
+
+ @Override
+ public Policy getPolicy(long policyID) throws IOException {
+ String rangerAdminUrl = rangerHttpsAddress +
+ OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT + policyID;
+
+ HttpsURLConnection conn = makeHttpsGetCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to get policy %d. " +
+ "Http response code: %d", policyID, conn.getResponseCode()));
+ }
+ String policyInfo = getResponseData(conn);
+ return jsonConverter.fromJson(policyInfo, Policy.class);
+ }
+
+ @Override
+ public void updatePolicy(long policyID, Policy policy) throws IOException {
+ String rangerAdminUrl = rangerHttpsAddress +
+ OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT + policyID;
+
+ HttpsURLConnection conn = makeHttpsPutCall(rangerAdminUrl,
+ jsonConverter.toJsonTree(policy));
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to update policy %d. " +
+ "Http response code: %d", policyID, conn.getResponseCode()));
+ }
+ }
+
+ @Override
+ public long createRole(Role role) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_ROLE_HTTP_ENDPOINT;
+
+ HttpsURLConnection conn = makeHttpsPostCall(rangerAdminUrl,
+ jsonConverter.toJsonTree(role).getAsJsonObject());
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to create role %s. " +
+ "Http response code: %d", role.getName(), conn.getResponseCode()));
+ }
+ String responseString = getResponseData(conn);
+ JsonObject jObject = new JsonParser().parse(responseString)
+ .getAsJsonObject();
+ return jObject.get("id").getAsLong();
+ }
+
+ @Override
+ public void deleteRole(long roleID) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_POLICY_HTTP_ENDPOINT
+ + roleID;
+ HttpsURLConnection conn = makeHttpsDeleteCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to delete role %d. " +
+ "Http response code: %d", roleID, conn.getResponseCode()));
+ }
+ }
+
+ @Override
+ public Map<Long, Role> getRoles() throws Exception {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_ROLE_HTTP_ENDPOINT;
+ HttpsURLConnection conn = makeHttpsGetCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to get all roles. " +
+ "Http response code: %d", conn.getResponseCode()));
+ }
+
+ String allRolesString = getResponseData(conn);
+ JsonArray rolesArrayJson =
+ new JsonParser().parse(allRolesString).getAsJsonArray();
+ Map<Long, Role> roles = new HashMap<>();
+ for (JsonElement roleJson: rolesArrayJson) {
+ long id = roleJson.getAsJsonObject().get("id").getAsLong();
+ roles.put(id, jsonConverter.fromJson(roleJson, Role.class));
+ }
+
+ return roles;
+ }
+
+ @Override
+ public Role getRole(long roleID) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_ROLE_HTTP_ENDPOINT + roleID;
+
+ HttpsURLConnection conn = makeHttpsGetCall(rangerAdminUrl);
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to get role %d. " +
+ "Http response code: %d", roleID, conn.getResponseCode()));
+ }
+ String roleInfo = getResponseData(conn);
+ return jsonConverter.fromJson(roleInfo, Role.class);
+ }
+
+ @Override
+ public void updateRole(long roleID, Role role) throws IOException {
+ String rangerAdminUrl =
+ rangerHttpsAddress + OZONE_OM_RANGER_ADMIN_ROLE_HTTP_ENDPOINT + roleID;
+
+ HttpsURLConnection conn = makeHttpsPutCall(rangerAdminUrl,
+ jsonConverter.toJsonTree(role));
+ if (!successfulResponseCode(conn.getResponseCode())) {
+ throw new IOException(String.format("Failed to update role %d. " +
+ "Http response code: %d", roleID, conn.getResponseCode()));
+ }
+ }
+
+ private HttpsURLConnection makeHttpsPutCall(String url, JsonElement content)
+ throws IOException {
+ HttpsURLConnection connection = makeBaseHttpsURLConnection(url);
+ connection.setRequestMethod("PUT");
+ return addJsonContentToConnection(connection, content);
+ }
+
+ private HttpsURLConnection makeHttpsPostCall(String url, JsonElement content)
+ throws IOException {
+ HttpsURLConnection connection = makeBaseHttpsURLConnection(url);
+ connection.setRequestMethod("POST");
+ return addJsonContentToConnection(connection, content);
+ }
+
+ private HttpsURLConnection addJsonContentToConnection(
+ HttpsURLConnection connection, JsonElement content) throws IOException {
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Content-Type", "application/json;");
+ try (OutputStream os = connection.getOutputStream()) {
+ byte[] input = content.toString().getBytes(StandardCharsets.UTF_8);
+ os.write(input, 0, input.length);
+ os.flush();
+ }
+
+ return connection;
+ }
+
+ private HttpsURLConnection makeHttpsGetCall(String urlString)
+ throws IOException {
+ HttpsURLConnection connection = makeBaseHttpsURLConnection(urlString);
+ connection.setRequestMethod("GET");
+ return connection;
+ }
+
+ private HttpsURLConnection makeHttpsDeleteCall(String urlString)
+ throws IOException {
+ HttpsURLConnection connection = makeBaseHttpsURLConnection(urlString);
+ connection.setRequestMethod("DELETE");
+ return connection;
+ }
+
+ private HttpsURLConnection makeBaseHttpsURLConnection(String urlString)
+ throws IOException {
+ URL url = new URL(urlString);
+ HttpsURLConnection urlConnection = (HttpsURLConnection)url.openConnection();
+ urlConnection.setConnectTimeout(connectionTimeout);
+ urlConnection.setReadTimeout(connectionRequestTimeout);
+ urlConnection.setRequestProperty("Accept", "application/json");
+ urlConnection.setRequestProperty("Authorization", authHeaderValue);
+
+ return urlConnection;
+ }
+
+ private String getResponseData(HttpsURLConnection urlConnection)
+ throws IOException {
+ StringBuilder response = new StringBuilder();
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(
+ urlConnection.getInputStream(), StandardCharsets.UTF_8))) {
+ String responseLine;
+ while ((responseLine = br.readLine()) != null) {
+ response.append(responseLine.trim());
+ }
+ }
+ return response.toString();
+ }
+
+ private boolean successfulResponseCode(long responseCode) {
+ return responseCode >= 200 && responseCode < 300;
+ }
+
+ /// SERIALIZATION ///
+
+ private final JsonDeserializer<Policy> policyDeserializer =
+ new JsonDeserializer<Policy>() {
- @Override
- public Policy deserialize(JsonElement jsonElement, Type type,
- JsonDeserializationContext jsonDeserializationContext)
- throws JsonParseException {
- JsonObject policyJson = jsonElement.getAsJsonObject();
- String name = policyJson.get("name").getAsString();
- Policy policy = new Policy(name);
- if (policyJson.has("description")) {
- policy.setDescription(policyJson.get("description").getAsString());
- }
- policy.setEnabled(policyJson.get("isEnabled").getAsBoolean());
-
- // Read volume, bucket, keys from json.
- JsonObject resourcesJson = policyJson.get("resources").getAsJsonObject();
- // All Ozone Ranger policies specify at least a volume.
- JsonObject jsonVolumeResource =
- resourcesJson.get("volume").getAsJsonObject();
- JsonArray volumes = jsonVolumeResource.get("values").getAsJsonArray();
- volumes.forEach(vol -> policy.addVolumes(vol.getAsString()));
-
- if (resourcesJson.has("bucket")) {
- JsonObject jsonBucketResource =
- resourcesJson.get("bucket").getAsJsonObject();
- JsonArray buckets = jsonBucketResource.get("values").getAsJsonArray();
- buckets.forEach(bucket -> policy.addBuckets(bucket.getAsString()));
- }
-
- if (resourcesJson.has("key")) {
- JsonObject jsonKeysResource =
- resourcesJson.get("key").getAsJsonObject();
- JsonArray keys = jsonKeysResource.get("values").getAsJsonArray();
- keys.forEach(key -> policy.addKeys(key.getAsString()));
- }
++ @Override public Policy deserialize(JsonElement jsonElement, Type type,
++ JsonDeserializationContext jsonDeserializationContext)
++ throws JsonParseException {
++ JsonObject policyJson = jsonElement.getAsJsonObject();
++ String name = policyJson.get("name").getAsString();
++ Policy policy = new Policy(name);
++ if (policyJson.has("description")) {
++ policy.setDescription(policyJson.get("description").getAsString());
++ }
++ policy.setEnabled(policyJson.get("isEnabled").getAsBoolean());
++
++ // Read volume, bucket, keys from json.
++ JsonObject resourcesJson =
++ policyJson.get("resources").getAsJsonObject();
++ // All Ozone Ranger policies specify at least a volume.
++ JsonObject jsonVolumeResource =
++ resourcesJson.get("volume").getAsJsonObject();
++ JsonArray volumes = jsonVolumeResource.get("values").getAsJsonArray();
++ volumes.forEach(vol -> policy.addVolumes(vol.getAsString()));
++
++ if (resourcesJson.has("bucket")) {
++ JsonObject jsonBucketResource =
++ resourcesJson.get("bucket").getAsJsonObject();
++ JsonArray buckets =
++ jsonBucketResource.get("values").getAsJsonArray();
++ buckets.forEach(bucket -> policy.addBuckets(bucket.getAsString()));
++ }
+
- // Read Roles and their ACLs.
- JsonArray policyItemsJson = policyJson.getAsJsonArray("policyItems");
- for (JsonElement policyItemElement: policyItemsJson) {
- JsonObject policyItemJson = policyItemElement.getAsJsonObject();
- JsonArray jsonRoles = policyItemJson.getAsJsonArray("roles");
- JsonArray jsonAclArray = policyItemJson.getAsJsonArray("accesses");
-
- for (JsonElement jsonAclElem: jsonAclArray) {
- JsonObject jsonAcl = jsonAclElem.getAsJsonObject();
- String aclType = jsonAcl.get("type").getAsString();
- Acl acl;
- if (jsonAcl.get("isAllowed").getAsBoolean()) {
- acl = Acl.allow(stringToAcl.get(aclType));
- } else {
- acl = Acl.deny(stringToAcl.get(aclType));
++ if (resourcesJson.has("key")) {
++ JsonObject jsonKeysResource =
++ resourcesJson.get("key").getAsJsonObject();
++ JsonArray keys = jsonKeysResource.get("values").getAsJsonArray();
++ keys.forEach(key -> policy.addKeys(key.getAsString()));
+ }
+
- for (JsonElement roleNameJson: jsonRoles) {
- policy.addRoleAcls(roleNameJson.getAsString(), acl);
++ // Read Roles and their ACLs.
++ JsonArray policyItemsJson = policyJson.getAsJsonArray("policyItems");
++ for (JsonElement policyItemElement : policyItemsJson) {
++ JsonObject policyItemJson = policyItemElement.getAsJsonObject();
++ JsonArray jsonRoles = policyItemJson.getAsJsonArray("roles");
++ JsonArray jsonAclArray = policyItemJson.getAsJsonArray("accesses");
++
++ for (JsonElement jsonAclElem : jsonAclArray) {
++ JsonObject jsonAcl = jsonAclElem.getAsJsonObject();
++ String aclType = jsonAcl.get("type").getAsString();
++ Acl acl;
++ if (jsonAcl.get("isAllowed").getAsBoolean()) {
++ acl = Acl.allow(stringToAcl.get(aclType));
++ } else {
++ acl = Acl.deny(stringToAcl.get(aclType));
++ }
++
++ for (JsonElement roleNameJson : jsonRoles) {
++ policy.addRoleAcls(roleNameJson.getAsString(), acl);
++ }
++ }
+ }
- }
- }
+
- return policy;
- }
- };
++ return policy;
++ }
++ };
+
+ private final JsonDeserializer<Role> roleDeserializer =
+ new JsonDeserializer<Role>() {
- @Override
- public Role deserialize(JsonElement jsonElement, Type type,
- JsonDeserializationContext jsonDeserializationContext)
- throws JsonParseException {
- JsonObject roleJson = jsonElement.getAsJsonObject();
- String name = roleJson.get("name").getAsString();
- Role role = new Role(name);
- if (roleJson.has("description")) {
- role.setDescription(roleJson.get("description").getAsString());
- }
- for (JsonElement jsonUser: roleJson.get("users").getAsJsonArray()) {
- String userName =
- jsonUser.getAsJsonObject().get("name").getAsString();
- role.addUsers(new BasicUserPrincipal(userName));
- }
++ @Override public Role deserialize(JsonElement jsonElement, Type type,
++ JsonDeserializationContext jsonDeserializationContext)
++ throws JsonParseException {
++ JsonObject roleJson = jsonElement.getAsJsonObject();
++ String name = roleJson.get("name").getAsString();
++ Role role = new Role(name);
++ if (roleJson.has("description")) {
++ role.setDescription(roleJson.get("description").getAsString());
++ }
++ for (JsonElement jsonUser : roleJson.get("users").getAsJsonArray()) {
++ String userName =
++ jsonUser.getAsJsonObject().get("name").getAsString();
++ role.addUsers(new BasicUserPrincipal(userName));
++ }
+
- return role;
- }
- };
++ return role;
++ }
++ };
+
+ private final JsonSerializer<Policy> policySerializer =
+ new JsonSerializer<Policy>() {
- @Override
- public JsonElement serialize(Policy javaPolicy, Type typeOfSrc,
- JsonSerializationContext context) {
- JsonObject jsonPolicy = new JsonObject();
- jsonPolicy.addProperty("name", javaPolicy.getName());
- jsonPolicy.addProperty("service", rangerService);
- jsonPolicy.addProperty("isEnabled", javaPolicy.isEnabled());
- if (javaPolicy.getDescription().isPresent()) {
- jsonPolicy.addProperty("description",
- javaPolicy.getDescription().get());
- }
++ @Override public JsonElement serialize(Policy javaPolicy,
++ Type typeOfSrc, JsonSerializationContext context) {
++ JsonObject jsonPolicy = new JsonObject();
++ jsonPolicy.addProperty("name", javaPolicy.getName());
++ jsonPolicy.addProperty("service", rangerService);
++ jsonPolicy.addProperty("isEnabled", javaPolicy.isEnabled());
++ if (javaPolicy.getDescription().isPresent()) {
++ jsonPolicy.addProperty("description",
++ javaPolicy.getDescription().get());
++ }
+
- // All resources under this policy are added to this object.
- JsonObject jsonResources = new JsonObject();
++ // All resources under this policy are added to this object.
++ JsonObject jsonResources = new JsonObject();
+
- // Add volumes. Ranger requires at least one volume to be specified.
- JsonArray jsonVolumeNameArray = new JsonArray();
- for (String volumeName: javaPolicy.getVolumes()) {
- jsonVolumeNameArray.add(new JsonPrimitive(volumeName));
- }
- JsonObject jsonVolumeResource = new JsonObject();
- jsonVolumeResource.add("values", jsonVolumeNameArray);
- jsonVolumeResource.addProperty("isRecursive", false);
- jsonVolumeResource.addProperty("isExcludes", false);
- jsonResources.add("volume", jsonVolumeResource);
-
- // Add buckets.
- JsonArray jsonBucketNameArray = new JsonArray();
- for (String bucketName: javaPolicy.getBuckets()) {
- jsonBucketNameArray.add(new JsonPrimitive(bucketName));
- }
++ // Add volumes. Ranger requires at least one volume to be specified.
++ JsonArray jsonVolumeNameArray = new JsonArray();
++ for (String volumeName : javaPolicy.getVolumes()) {
++ jsonVolumeNameArray.add(new JsonPrimitive(volumeName));
++ }
++ JsonObject jsonVolumeResource = new JsonObject();
++ jsonVolumeResource.add("values", jsonVolumeNameArray);
++ jsonVolumeResource.addProperty("isRecursive", false);
++ jsonVolumeResource.addProperty("isExcludes", false);
++ jsonResources.add("volume", jsonVolumeResource);
++
++ // Add buckets.
++ JsonArray jsonBucketNameArray = new JsonArray();
++ for (String bucketName : javaPolicy.getBuckets()) {
++ jsonBucketNameArray.add(new JsonPrimitive(bucketName));
++ }
+
- if (jsonBucketNameArray.size() > 0) {
- JsonObject jsonBucketResource = new JsonObject();
- jsonBucketResource.add("values", jsonBucketNameArray);
- jsonBucketResource.addProperty("isRecursive", false);
- jsonBucketResource.addProperty("isExcludes", false);
- jsonResources.add("bucket", jsonBucketResource);
- }
++ if (jsonBucketNameArray.size() > 0) {
++ JsonObject jsonBucketResource = new JsonObject();
++ jsonBucketResource.add("values", jsonBucketNameArray);
++ jsonBucketResource.addProperty("isRecursive", false);
++ jsonBucketResource.addProperty("isExcludes", false);
++ jsonResources.add("bucket", jsonBucketResource);
++ }
+
- // Add keys.
- JsonArray jsonKeyNameArray = new JsonArray();
- for (String keyName: javaPolicy.getKeys()) {
- jsonKeyNameArray.add(new JsonPrimitive(keyName));
- }
- if (jsonKeyNameArray.size() > 0) {
- JsonObject jsonKeyResource = new JsonObject();
- jsonKeyResource.add("values", jsonKeyNameArray);
- jsonKeyResource.addProperty("isRecursive", false);
- jsonKeyResource.addProperty("isExcludes", false);
- jsonResources.add("key", jsonKeyResource);
- }
++ // Add keys.
++ JsonArray jsonKeyNameArray = new JsonArray();
++ for (String keyName : javaPolicy.getKeys()) {
++ jsonKeyNameArray.add(new JsonPrimitive(keyName));
++ }
++ if (jsonKeyNameArray.size() > 0) {
++ JsonObject jsonKeyResource = new JsonObject();
++ jsonKeyResource.add("values", jsonKeyNameArray);
++ jsonKeyResource.addProperty("isRecursive", false);
++ jsonKeyResource.addProperty("isExcludes", false);
++ jsonResources.add("key", jsonKeyResource);
++ }
+
- jsonPolicy.add("resources", jsonResources);
-
- // Add roles and their acls to the policy.
- JsonArray jsonPolicyItemArray = new JsonArray();
-
- // Make a new policy item for each role in the map.
- Map<String, Collection<Acl>> roleAcls = javaPolicy.getRoleAcls();
- for (Map.Entry<String, Collection<Acl>> entry: roleAcls.entrySet()) {
- // Add role to the policy item.
- String roleName = entry.getKey();
- JsonObject jsonPolicyItem = new JsonObject();
- JsonArray jsonRoles = new JsonArray();
- jsonRoles.add(new JsonPrimitive(roleName));
- jsonPolicyItem.add("roles", jsonRoles);
-
- // Add acls to the policy item.
- JsonArray jsonAclArray = new JsonArray();
- for (Acl acl: entry.getValue()) {
- JsonObject jsonAcl = new JsonObject();
- jsonAcl.addProperty("type",
- aclToString.get(acl.getAclType()));
- jsonAcl.addProperty("isAllowed", acl.isAllowed());
- jsonAclArray.add(jsonAcl);
- jsonPolicyItem.add("accesses", jsonAclArray);
- }
- jsonPolicyItemArray.add(jsonPolicyItem);
- }
- jsonPolicy.add("policyItems", jsonPolicyItemArray);
++ jsonPolicy.add("resources", jsonResources);
++
++ // Add roles and their acls to the policy.
++ JsonArray jsonPolicyItemArray = new JsonArray();
++
++ // Make a new policy item for each role in the map.
++ Map<String, Collection<Acl>> roleAcls = javaPolicy.getRoleAcls();
++ for (Map.Entry<String, Collection<Acl>> entry : roleAcls.entrySet()) {
++ // Add role to the policy item.
++ String roleName = entry.getKey();
++ JsonObject jsonPolicyItem = new JsonObject();
++ JsonArray jsonRoles = new JsonArray();
++ jsonRoles.add(new JsonPrimitive(roleName));
++ jsonPolicyItem.add("roles", jsonRoles);
++
++ // Add acls to the policy item.
++ JsonArray jsonAclArray = new JsonArray();
++ for (Acl acl : entry.getValue()) {
++ JsonObject jsonAcl = new JsonObject();
++ jsonAcl.addProperty("type", aclToString.get(acl.getAclType()));
++ jsonAcl.addProperty("isAllowed", acl.isAllowed());
++ jsonAclArray.add(jsonAcl);
++ jsonPolicyItem.add("accesses", jsonAclArray);
++ }
++ jsonPolicyItemArray.add(jsonPolicyItem);
++ }
++ jsonPolicy.add("policyItems", jsonPolicyItemArray);
+
- return jsonPolicy;
- }
- };
++ return jsonPolicy;
++ }
++ };
+
+ private final JsonSerializer<Role> roleSerializer =
+ new JsonSerializer<Role>() {
- @Override
- public JsonElement serialize(Role javaRole, Type typeOfSrc,
- JsonSerializationContext context) {
- JsonObject jsonRole = new JsonObject();
- jsonRole.addProperty("name", javaRole.getName());
-
- JsonArray jsonUserArray = new JsonArray();
- for (BasicUserPrincipal javaUser: javaRole.getUsers()) {
- jsonUserArray.add(jsonConverter.toJsonTree(javaUser));
- }
++ @Override public JsonElement serialize(Role javaRole, Type typeOfSrc,
++ JsonSerializationContext context) {
++ JsonObject jsonRole = new JsonObject();
++ jsonRole.addProperty("name", javaRole.getName());
++
++ JsonArray jsonUserArray = new JsonArray();
++ for (BasicUserPrincipal javaUser : javaRole.getUsers()) {
++ jsonUserArray.add(jsonConverter.toJsonTree(javaUser));
++ }
+
- jsonRole.add("users", jsonUserArray);
- return jsonRole;
- }
- };
++ jsonRole.add("users", jsonUserArray);
++ return jsonRole;
++ }
++ };
+
+ private final JsonSerializer<BasicUserPrincipal> userSerializer =
+ new JsonSerializer<BasicUserPrincipal>() {
- @Override
- public JsonElement serialize(BasicUserPrincipal user, Type typeOfSrc,
- JsonSerializationContext context) {
- JsonObject jsonMember = new JsonObject();
- jsonMember.addProperty("name", user.getName());
- jsonMember.addProperty("isAdmin", false);
- return jsonMember;
- }
- };
++ @Override public JsonElement serialize(BasicUserPrincipal user,
++ Type typeOfSrc, JsonSerializationContext context) {
++ JsonObject jsonMember = new JsonObject();
++ jsonMember.addProperty("name", user.getName());
++ jsonMember.addProperty("isAdmin", false);
++ return jsonMember;
++ }
++ };
+}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 7a48f28917,29d5a09f68..6248f27be0
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@@ -86,6 -80,6 +87,8 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteTenantRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteTenantResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressResponse;
@@@ -98,6 -92,6 +101,8 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretResponse;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoVolumeRequest;
@@@ -144,18 -136,12 +149,24 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RevokeS3SecretRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetVolumePropertyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantAssignAdminRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantAssignUserAccessIdRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantAssignUserAccessIdResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoResponse;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantRevokeAdminRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantRevokeUserAccessIdRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
diff --cc hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot
index 7493c65ea9,f6de86add8..acecbab5f8
--- a/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot
@@@ -30,30 -30,7 +31,35 @@@ Create a volume and bucke
Should not contain ${output} Failed
${output} = Execute ozone sh bucket create /${PREFIX}-volume/${PREFIX}-bucket
Should not contain ${output} Failed
+
+ Create key
- ${output} = Execute ozone sh key put /${PREFIX}-volume/${PREFIX}-bucket/${PREFIX}-key /opt/hadoop/NOTICE.txt
+ Execute and checkrc echo "${PREFIX}: key created using Ozone Shell" > /tmp/sourcekey 0
+ ${output} = Execute ozone sh key put /${PREFIX}-volume/${PREFIX}-bucket/${PREFIX}-key /tmp/sourcekey
Should not contain ${output} Failed
+ Execute and checkrc rm /tmp/sourcekey 0
+
- Create a bucket and key in volume s3v
++Create a bucket in s3v volume
++ [Tags] create-volume-and-bucket
+ ${output} = Execute ozone sh bucket create /s3v/${PREFIX}-bucket
+ Should not contain ${output} Failed
++
++Create key in the bucket in s3v volume
+ Execute and checkrc echo "${PREFIX}: another key created using Ozone Shell" > /tmp/sourcekey 0
+ ${output} = Execute ozone sh key put /s3v/${PREFIX}-bucket/key1-shell /tmp/sourcekey
+ Should not contain ${output} Failed
+ Execute and checkrc rm /tmp/sourcekey 0
+
+Setup credentials for S3
+ # TODO: Run "Setup secure v4 headers" instead when security is enabled
+ Run Keyword Setup dummy credentials for S3
+
+Try to create a bucket using S3 API
+ # Note: S3 API does not return error if the bucket already exists
+ ${output} = Create bucket with name ${PREFIX}-bucket
+ Should Be Equal ${output} ${None}
+
+Create key using S3 API
+ Execute and checkrc echo "${PREFIX}: key created using S3 API" > /tmp/sourcekey 0
+ ${output} = Execute AWSS3APICli and checkrc put-object --bucket ${PREFIX}-bucket --key key2-s3api --body /tmp/sourcekey 0
+ Should not contain ${output} error
+ Execute and checkrc rm /tmp/sourcekey 0
diff --cc hadoop-ozone/dist/src/shell/ozone/ozone
index 12a3997675,72be8cfb45..8a0d81bfa7
--- a/hadoop-ozone/dist/src/shell/ozone/ozone
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone
@@@ -168,12 -167,9 +168,13 @@@ function ozonecmd_cas
s3g)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway'
+ OZONE_S3G_OPTS="${OZONE_S3G_OPTS} -Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties"
OZONE_RUN_ARTIFACT_NAME="ozone-s3gateway"
;;
+ tenant)
+ OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.tenant.TenantShell
+ OZONE_RUN_ARTIFACT_NAME="ozone-tools"
+ ;;
csi)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.csi.CsiServer'
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 6c20625946,ef561e657b..15eda5a4b9
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -68,15 -68,11 +68,15 @@@ public class TestOzoneConfigurationFiel
configurationPropsToSkipCompare
.add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION);
configurationPropsToSkipCompare
- .add(OzoneConfigKeys.OZONE_OM_CLIENT_PROTOCOL_VERSION_KEY);
+ .add(OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY);
configurationPropsToSkipCompare
- .add(OzoneConfigKeys.OZONE_OM_CLIENT_PROTOCOL_VERSION);
+ .add(OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_DEFAULT);
// This property is tested in TestHttpServer2 instead
xmlPropsToSkipCompare.add(HttpServer2.HTTP_IDLE_TIMEOUT_MS_KEY);
+
+ // TODO: Remove this once ranger configs are finalized in HDDS-5836
+ configurationPrefixToSkipCompare.add("ozone.om.ranger");
+
addPropertiesNotInXml();
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
index a8aabc0593,0000000000..138ad57470
mode 100644,000000..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java
@@@ -1,244 -1,0 +1,244 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.multitenant;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.om.protocol.S3Auth;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.LambdaTestUtils;
+import org.apache.ozone.test.LambdaTestUtils.VoidCallable;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
+import static org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
+
+/**
+ * Tests that S3 requests for a tenant are directed to that tenant's volume,
+ * and that users not belonging to a tenant are directed to the default S3
+ * volume.
+ */
+public class TestMultiTenantVolume {
+ private static MiniOzoneCluster cluster;
+ private static String s3VolumeName;
+
+ private static final String TENANT_ID = "tenant";
+ private static final String USER_PRINCIPAL = "username";
+ private static final String BUCKET_NAME = "bucket";
+ private static final String ACCESS_ID = "tenant$username";
+
+ @BeforeClass
+ public static void initClusterProvider() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(
+ OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true);
+ MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
+ .withoutDatanodes()
+ .setOmLayoutVersion(OMLayoutFeature.INITIAL_VERSION.layoutVersion());
+ cluster = builder.build();
+ s3VolumeName = HddsClientUtils.getDefaultS3VolumeName(conf);
+
+ preFinalizationChecks(getStoreForAccessID(ACCESS_ID));
+ finalizeOMUpgrade();
+ }
+
+ @AfterClass
+ public static void shutdownClusterProvider() {
+ cluster.shutdown();
+ }
+
+ private static void expectFailurePreFinalization(VoidCallable eval)
+ throws Exception {
+ LambdaTestUtils.intercept(OMException.class,
+ "cannot be invoked before finalization", eval);
+ }
+
+ /**
+ * Perform sanity checks before triggering upgrade finalization.
+ */
+ private static void preFinalizationChecks(ObjectStore store)
+ throws Exception {
+
+ // None of the tenant APIs is usable before the upgrade finalization step
+ expectFailurePreFinalization(
+ store::listTenant);
+ expectFailurePreFinalization(() ->
+ store.listUsersInTenant(TENANT_ID, ""));
+ expectFailurePreFinalization(() ->
+ store.tenantGetUserInfo(USER_PRINCIPAL));
+ expectFailurePreFinalization(() ->
+ store.createTenant(TENANT_ID));
+ expectFailurePreFinalization(() ->
+ store.tenantAssignUserAccessId(USER_PRINCIPAL, TENANT_ID, ACCESS_ID));
+ expectFailurePreFinalization(() ->
+ store.tenantAssignAdmin(USER_PRINCIPAL, TENANT_ID, true));
+ expectFailurePreFinalization(() ->
+ store.tenantRevokeAdmin(ACCESS_ID, TENANT_ID));
+ expectFailurePreFinalization(() ->
+ store.tenantRevokeUserAccessId(ACCESS_ID));
+ expectFailurePreFinalization(() ->
+ store.deleteTenant(TENANT_ID));
+
+ // S3 get/set/revoke secret APIs still work before finalization
+ final String accessId = "testUser1accessId1";
+ S3SecretValue s3SecretValue = store.getS3Secret(accessId);
+ Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey());
+ final String setSecret = "testsecret";
+ s3SecretValue = store.setS3Secret(accessId, setSecret);
+ Assert.assertEquals(accessId, s3SecretValue.getAwsAccessKey());
+ Assert.assertEquals(setSecret, s3SecretValue.getAwsSecret());
+ store.revokeS3Secret(accessId);
+ }
+
+ /**
+ * Trigger OM upgrade finalization from the client and block until completion
+ * (status FINALIZATION_DONE).
+ */
+ private static void finalizeOMUpgrade()
+ throws IOException, InterruptedException, TimeoutException {
+
+ // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
+ final OzoneManagerProtocol client = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
+ final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
+ UpgradeFinalizer.StatusAndMessages finalizationResponse =
+ client.finalizeUpgrade(upgradeClientID);
+
+ // The status should transition as soon as the client call above returns
+ Assert.assertTrue(isStarting(finalizationResponse.status()));
+
+ // Wait for the finalization to be marked as done.
+ // 10s timeout should be plenty.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ final UpgradeFinalizer.StatusAndMessages progress =
+ client.queryUpgradeFinalizationProgress(
+ upgradeClientID, false, false);
+ return isDone(progress.status());
+ } catch (IOException e) {
+ Assert.fail("Unexpected exception while waiting for "
+ + "the OM upgrade to finalize: " + e.getMessage());
+ }
+ return false;
+ }, 500, 10000);
+ }
+
+ @Test
+ public void testDefaultS3Volume() throws Exception {
+ final String bucketName = "bucket";
+
+ // Default client not belonging to a tenant should end up in the S3 volume.
+ ObjectStore store = cluster.getClient().getObjectStore();
+ Assert.assertEquals(s3VolumeName, store.getS3Volume().getName());
+
+ // Create bucket.
+ store.createS3Bucket(bucketName);
+ Assert.assertEquals(s3VolumeName,
+ store.getS3Bucket(bucketName).getVolumeName());
+
+ // Delete bucket.
+ store.deleteS3Bucket(bucketName);
+ assertS3BucketNotFound(store, bucketName);
+ }
+
+ @Test
+ public void testS3TenantVolume() throws Exception {
+
+ ObjectStore store = getStoreForAccessID(ACCESS_ID);
+
+ store.createTenant(TENANT_ID);
+ store.tenantAssignUserAccessId(USER_PRINCIPAL, TENANT_ID, ACCESS_ID);
+
+ // S3 volume pointed to by the store should be for the tenant.
+ Assert.assertEquals(TENANT_ID, store.getS3Volume().getName());
+
+ // Create bucket in the tenant volume.
+ store.createS3Bucket(BUCKET_NAME);
+ OzoneBucket bucket = store.getS3Bucket(BUCKET_NAME);
+ Assert.assertEquals(TENANT_ID, bucket.getVolumeName());
+
+ // A different user should not see bucket, since they will be directed to
+ // the s3 volume.
+ ObjectStore store2 = getStoreForAccessID(UUID.randomUUID().toString());
+ assertS3BucketNotFound(store2, BUCKET_NAME);
+
+ // Delete bucket.
+ store.deleteS3Bucket(BUCKET_NAME);
+ assertS3BucketNotFound(store, BUCKET_NAME);
+
+ store.tenantRevokeUserAccessId(ACCESS_ID);
+ store.deleteTenant(TENANT_ID);
+ }
+
+ /**
+ * Checks that the bucket is not found using
+ * {@link ObjectStore#getS3Bucket} and the designated S3 volume pointed to
+ * by the ObjectStore.
+ */
+ private void assertS3BucketNotFound(ObjectStore store, String bucketName)
+ throws IOException {
+ try {
+ store.getS3Bucket(bucketName);
+ } catch (OMException ex) {
+ if (ex.getResult() != OMException.ResultCodes.BUCKET_NOT_FOUND) {
+ throw ex;
+ }
+ }
+
+ try {
+ OzoneVolume volume = store.getS3Volume();
+ volume.getBucket(bucketName);
+ } catch (OMException ex) {
+ if (ex.getResult() != OMException.ResultCodes.BUCKET_NOT_FOUND) {
+ throw ex;
+ }
+ }
+ }
+
+ private static ObjectStore getStoreForAccessID(String accessID)
+ throws IOException {
+ // Cluster provider will modify our provided configuration. We must use
+ // this version to build the client.
+ OzoneConfiguration conf = cluster.getOzoneManager().getConfiguration();
+ // Manually construct an object store instead of using the cluster
+ // provided one so we can specify the access ID.
+ RpcClient client = new RpcClient(conf, null);
+ // userPrincipal is set to be the same as accessId for the test
- client.setTheadLocalS3Auth(
++ client.setThreadLocalS3Auth(
+ new S3Auth("unused1", "unused2", accessID, accessID));
+ return new ObjectStore(conf, client);
+ }
+}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 83708c20d0,1a0e7d461c..445218f530
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -67,9 -67,7 +67,10 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+ import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index ef688ee589,ef61417c33..0df19af147
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@@ -182,43 -179,12 +182,43 @@@ public class OMDBDefinition implements
public static final DBColumnFamilyDefinition<String, String>
META_TABLE = new DBColumnFamilyDefinition<>(
- OmMetadataManagerImpl.META_TABLE,
- String.class,
- new StringCodec(),
- String.class,
- new StringCodec());
+ OmMetadataManagerImpl.META_TABLE,
+ String.class,
+ new StringCodec(),
+ String.class,
+ new StringCodec());
+ // Tables for multi-tenancy
+
+ public static final DBColumnFamilyDefinition<String, OmDBAccessIdInfo>
+ TENANT_ACCESS_ID_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_ACCESS_ID_TABLE,
+ String.class, // accessId
+ new StringCodec(),
+ OmDBAccessIdInfo.class, // tenantId, secret, principal
+ new OmDBAccessIdInfoCodec());
+
+ public static final DBColumnFamilyDefinition<String, OmDBUserPrincipalInfo>
+ PRINCIPAL_TO_ACCESS_IDS_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.PRINCIPAL_TO_ACCESS_IDS_TABLE,
+ String.class, // User principal
+ new StringCodec(),
+ OmDBUserPrincipalInfo.class, // List of accessIds
+ new OmDBUserPrincipalInfoCodec());
+
+ public static final DBColumnFamilyDefinition<String, OmDBTenantState>
+ TENANT_STATE_TABLE =
+ new DBColumnFamilyDefinition<>(
+ OmMetadataManagerImpl.TENANT_STATE_TABLE,
+ String.class, // tenantId (tenant name)
+ new StringCodec(),
+ OmDBTenantState.class,
+ new OmDBTenantStateCodec());
+
+ // End tables for S3 multi-tenancy
+
@Override
public String getName() {
return OzoneConsts.OM_DB_NAME;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 114a528ad4,e9a4be8acc..b425ea4424
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@@ -176,42 -174,85 +181,99 @@@ public final class OzoneManagerRatisUti
return new OMPrepareRequest(omRequest);
case CancelPrepare:
return new OMCancelPrepareRequest(omRequest);
+ case SetS3Secret:
+ return new OMSetSecretRequest(omRequest);
case RevokeS3Secret:
return new S3RevokeSecretRequest(omRequest);
+ case PurgeKeys:
+ return new OMKeyPurgeRequest(omRequest);
+ case PurgePaths:
+ return new OMPathsPurgeRequestWithFSO(omRequest);
+ case CreateTenant:
+ return new OMTenantCreateRequest(omRequest);
+ case DeleteTenant:
+ return new OMTenantDeleteRequest(omRequest);
+ case TenantAssignUserAccessId:
+ return new OMTenantAssignUserAccessIdRequest(omRequest);
+ case TenantRevokeUserAccessId:
+ return new OMTenantRevokeUserAccessIdRequest(omRequest);
+ case TenantAssignAdmin:
+ return new OMTenantAssignAdminRequest(omRequest);
+ case TenantRevokeAdmin:
+ return new OMTenantRevokeAdminRequest(omRequest);
- /**
- * Following key requests will be created in {@link OMKeyRequestFactory}.
+ /*
+ * Key requests that can have multiple variants based on the bucket layout
+ * should be created using {@link BucketLayoutAwareOMKeyRequestFactory}.
*/
case CreateDirectory:
+ keyArgs = omRequest.getCreateDirectoryRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case CreateFile:
+ keyArgs = omRequest.getCreateFileRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case CreateKey:
+ keyArgs = omRequest.getCreateKeyRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case AllocateBlock:
+ keyArgs = omRequest.getAllocateBlockRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case CommitKey:
+ keyArgs = omRequest.getCommitKeyRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case DeleteKey:
+ keyArgs = omRequest.getDeleteKeyRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case DeleteKeys:
+ OzoneManagerProtocolProtos.DeleteKeyArgs deleteKeyArgs =
+ omRequest.getDeleteKeysRequest()
+ .getDeleteKeys();
+ volumeName = deleteKeyArgs.getVolumeName();
+ bucketName = deleteKeyArgs.getBucketName();
+ break;
case RenameKey:
+ keyArgs = omRequest.getRenameKeyRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case RenameKeys:
- case PurgeKeys:
- case PurgePaths:
+ OzoneManagerProtocolProtos.RenameKeysArgs renameKeysArgs =
+ omRequest.getRenameKeysRequest().getRenameKeysArgs();
+ volumeName = renameKeysArgs.getVolumeName();
+ bucketName = renameKeysArgs.getBucketName();
+ break;
case InitiateMultiPartUpload:
+ keyArgs = omRequest.getInitiateMultiPartUploadRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case CommitMultiPartUpload:
+ keyArgs = omRequest.getCommitMultiPartUploadRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case AbortMultiPartUpload:
+ keyArgs = omRequest.getAbortMultiPartUploadRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
case CompleteMultiPartUpload:
- return OMKeyRequestFactory.createRequest(omRequest, ozoneManager);
+ keyArgs = omRequest.getCompleteMultiPartUploadRequest().getKeyArgs();
+ volumeName = keyArgs.getVolumeName();
+ bucketName = keyArgs.getBucketName();
+ break;
default:
throw new IllegalStateException("Unrecognized write command " +
"type request" + cmdType);
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/OMSetSecretRequest.java
index a1b5702a89,0000000000..63318f0161
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/OMSetSecretRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/OMSetSecretRequest.java
@@@ -1,204 -1,0 +1,207 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.s3.security;
+
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.s3.security.OMSetSecretResponse;
- import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.*;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretRequest;
++import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetS3SecretResponse;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_SECRET_LOCK;
+
+/**
+ * Handles SetSecret request.
+ */
+public class OMSetSecretRequest extends OMClientRequest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMSetSecretRequest.class);
+
+ public OMSetSecretRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ @Override
+ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
+ final OMMetadataManager omMetadataManager =
+ ozoneManager.getMetadataManager();
+
+ final SetS3SecretRequest request =
+ getOmRequest().getSetS3SecretRequest();
+
+ final String accessId = request.getAccessId();
+
+ // First check accessId existence
+ final OmDBAccessIdInfo accessIdInfo = omMetadataManager
+ .getTenantAccessIdTable().get(accessId);
+
+ if (accessIdInfo == null) {
+ // Check (old) S3SecretTable
+ if (omMetadataManager.getS3SecretTable().get(accessId) == null) {
+ throw new OMException("accessId '" + accessId + "' not found.",
+ OMException.ResultCodes.ACCESS_ID_NOT_FOUND);
+ }
+ }
+
+ // Secret should not be empty
+ final String secretKey = request.getSecretKey();
+ if (StringUtils.isEmpty(secretKey)) {
+ throw new OMException("Secret key should not be empty",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+
+ if (secretKey.length() < OzoneConsts.S3_SECRET_KEY_MIN_LENGTH) {
+ throw new OMException("Secret key length should be at least " +
+ OzoneConsts.S3_SECRET_KEY_MIN_LENGTH + " characters",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+
+ // TODO: Check if secretKey matches other requirements? e.g. combination
+
+ final UserGroupInformation ugi = ProtobufRpcEngine.Server.getRemoteUser();
+ final String username = ugi.getUserName();
+
+ // Permission check. To pass the check, any one of the following conditions
+ // shall be satisfied:
+ // 1. username matches accessId exactly
+ // 2. user is an OM admin
+ // 3. user is assigned to a tenant under this accessId
+ // 4. user is an admin of the tenant where the accessId is assigned
+
+ if (!username.equals(accessId) && !ozoneManager.isAdmin(ugi)) {
+ // Attempt to retrieve tenant info using the accessId
+ if (!ozoneManager.getMultiTenantManager()
+ .isUserAccessIdPrincipalOrTenantAdmin(accessId, ugi)) {
+ throw new OMException("Permission denied. Requested accessId '" +
+ accessId + "' and user doesn't satisfy any of:\n" +
+ "1) accessId match current username: '" + username + "';\n" +
+ "2) is an OM admin;\n" +
+ "3) user is assigned to a tenant under this accessId;\n" +
+ "4) user is an admin of the tenant where the accessId is " +
+ "assigned", OMException.ResultCodes.PERMISSION_DENIED);
+ }
+ }
+
+ return getOmRequest();
+ }
+
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long transactionLogIndex,
+ OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) {
+
+ OMClientResponse omClientResponse = null;
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
+ boolean acquiredLock = false;
+ IOException exception = null;
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+
+ final SetS3SecretRequest request = getOmRequest().getSetS3SecretRequest();
+ final String accessId = request.getAccessId();
+ final String secretKey = request.getSecretKey();
+
+ try {
+ acquiredLock = omMetadataManager.getLock().acquireWriteLock(
+ S3_SECRET_LOCK, accessId);
+
+ // Intentionally set to final so they can only be set once.
+ final S3SecretValue newS3SecretValue;
+
+ // Update legacy S3SecretTable, if the accessId entry exists
+ if (omMetadataManager.getS3SecretTable().get(accessId) != null) {
+ // accessId found in S3SecretTable. Update S3SecretTable
+ LOG.debug("Updating S3SecretTable cache entry");
+ // Update S3SecretTable cache entry in this case
+ newS3SecretValue = new S3SecretValue(accessId, secretKey);
+
+ omMetadataManager.getS3SecretTable().addCacheEntry(
+ new CacheKey<>(accessId),
+ new CacheValue<>(Optional.of(newS3SecretValue),
+ transactionLogIndex));
+ } else {
+ // If S3SecretTable is not updated, throw ACCESS_ID_NOT_FOUND exception.
+ throw new OMException("accessId '" + accessId + "' not found.",
+ OMException.ResultCodes.ACCESS_ID_NOT_FOUND);
+ }
+
+ // Compose response
+ final SetS3SecretResponse.Builder setSecretResponse =
+ SetS3SecretResponse.newBuilder()
+ .setAccessId(accessId)
+ .setSecretKey(secretKey);
+
+ omClientResponse = new OMSetSecretResponse(accessId, newS3SecretValue,
+ omResponse.setSetS3SecretResponse(setSecretResponse).build());
+
+ } catch (IOException ex) {
+ exception = ex;
+ omClientResponse = new OMSetSecretResponse(
+ createErrorOMResponse(omResponse, ex));
+ } finally {
+ addResponseToDoubleBuffer(transactionLogIndex, omClientResponse,
+ ozoneManagerDoubleBufferHelper);
+ if (acquiredLock) {
+ omMetadataManager.getLock().releaseWriteLock(S3_SECRET_LOCK,
+ accessId);
+ }
+ }
+
+ final Map<String, String> auditMap = new HashMap<>();
+ auditMap.put(OzoneConsts.S3_SETSECRET_USER, accessId);
+
+ // audit log
+ auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(
+ OMAction.SET_S3_SECRET, auditMap,
+ exception, getOmRequest().getUserInfo()));
+
+ if (exception == null) {
+ LOG.debug("Success: SetSecret for accessKey '{}'", accessId);
+ } else {
+ LOG.error("Failed to SetSecret for accessKey '{}': {}",
+ accessId, exception);
+ }
+ return omClientResponse;
+ }
+
+}
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 049ad61738,b6a928f289..32285d282c
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@@ -22,8 -24,16 +24,15 @@@ import javax.ws.rs.core.Context
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
+ import java.util.Map;
import java.util.function.Function;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+ import org.apache.hadoop.ozone.audit.AuditAction;
+ import org.apache.hadoop.ozone.audit.AuditEventStatus;
+ import org.apache.hadoop.ozone.audit.AuditLogger;
+ import org.apache.hadoop.ozone.audit.AuditLoggerType;
+ import org.apache.hadoop.ozone.audit.AuditMessage;
+ import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index fc727acb89,a852a3c2ab..838e86ea85
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@@ -189,9 -203,10 +205,10 @@@ public class ObjectEndpoint extends End
if (copyHeader != null) {
//Copy object, as copy source available.
+ s3GAction = S3GAction.COPY_OBJECT;
CopyObjectResponse copyObjectResponse = copyObject(
- copyHeader, bucketName, keyPath, replicationType,
- replicationFactor, storageTypeDefault);
+ copyHeader, bucketName, keyPath, replicationConfig,
+ storageTypeDefault);
return Response.status(Status.OK).entity(copyObjectResponse).header(
"Connection", "close").build();
}
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
index 0000000000,a598bdea60..5be8092e48
mode 000000,100644..100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
@@@ -1,0 -1,156 +1,158 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+ package org.apache.hadoop.ozone.s3;
+
+ import org.apache.commons.io.FileUtils;
+ import org.apache.commons.lang3.RandomStringUtils;
++import org.apache.hadoop.hdds.client.ReplicationConfig;
+ import org.apache.hadoop.hdds.client.ReplicationFactor;
+ import org.apache.hadoop.hdds.client.ReplicationType;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.ozone.OzoneConsts;
+ import org.apache.hadoop.ozone.client.OzoneBucket;
+ import org.apache.hadoop.ozone.client.OzoneClient;
+ import org.apache.hadoop.ozone.client.OzoneClientStub;
+ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+ import org.apache.hadoop.ozone.s3.endpoint.BucketEndpoint;
+ import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
+ import org.apache.hadoop.ozone.s3.endpoint.RootEndpoint;
+ import org.junit.AfterClass;
+ import org.junit.Before;
+ import org.junit.Test;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.File;
+ import java.io.IOException;
+ import java.util.HashMap;
+ import java.util.List;
+
+ import static java.nio.charset.StandardCharsets.UTF_8;
+ import static org.junit.Assert.assertEquals;
+
+ /**
+ * Tests for S3Gateway Audit Log.
+ */
+ public class TestS3GatewayAuditLog {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestS3GatewayAuditLog.class.getName());
+
+ static {
+ System.setProperty("log4j.configurationFile", "auditlog.properties");
+ System.setProperty("log4j2.contextSelector",
+ "org.apache.logging.log4j.core.async.AsyncLoggerContextSelector");
+ }
+
+ private String bucketName = OzoneConsts.BUCKET;
+ private OzoneClient clientStub;
+ private BucketEndpoint bucketEndpoint;
+ private RootEndpoint rootEndpoint;
+ private ObjectEndpoint keyEndpoint;
+ private OzoneBucket bucket;
+
+ @Before
+ public void setup() throws Exception {
+
+ clientStub = new OzoneClientStub();
+ clientStub.getObjectStore().createS3Bucket(bucketName);
+ bucket = clientStub.getObjectStore().getS3Bucket(bucketName);
+
+ bucketEndpoint = new BucketEndpoint();
+ bucketEndpoint.setClient(clientStub);
+
+ rootEndpoint = new RootEndpoint();
+ rootEndpoint.setClient(clientStub);
+
+ keyEndpoint = new ObjectEndpoint();
+ keyEndpoint.setClient(clientStub);
+ keyEndpoint.setOzoneConfiguration(new OzoneConfiguration());
+
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ File file = new File("audit.log");
+ if (FileUtils.deleteQuietly(file)) {
+ LOG.info("{} has been deleted as all tests have completed.",
+ file.getName());
+ } else {
+ LOG.info("audit.log could not be deleted.");
+ }
+ }
+
+ @Test
+ public void testHeadBucket() throws Exception {
+ bucketEndpoint.head(bucketName);
+ String expected = "INFO | S3GAudit | ? | user=null | ip=null | " +
+ "op=HEAD_BUCKET {bucket=bucket} | ret=SUCCESS";
+ verifyLog(expected);
+ }
+
+ @Test
+ public void testListBucket() throws Exception {
+
+ rootEndpoint.get().getEntity();
+ String expected = "INFO | S3GAudit | ? | user=null | ip=null | " +
+ "op=LIST_S3_BUCKETS {} | ret=SUCCESS";
+ verifyLog(expected);
+ }
+
+ @Test
+ public void testHeadObject() throws Exception {
+ String value = RandomStringUtils.randomAlphanumeric(32);
+ OzoneOutputStream out = bucket.createKey("key1",
- value.getBytes(UTF_8).length, ReplicationType.RATIS,
- ReplicationFactor.ONE, new HashMap<>());
++ value.getBytes(UTF_8).length,
++ ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
++ ReplicationFactor.ONE), new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+
+
+ keyEndpoint.head(bucketName, "key1");
+ String expected = "INFO | S3GAudit | ? | user=null | ip=null | " +
+ "op=HEAD_KEY {bucket=bucket, keyPath=key1} | ret=SUCCESS";
+ verifyLog(expected);
+
+ }
+
+ private void verifyLog(String expectedString) throws IOException {
+ File file = new File("audit.log");
+ List<String> lines = FileUtils.readLines(file, (String)null);
+ final int retry = 5;
+ int i = 0;
+ while (lines.isEmpty() && i < retry) {
+ lines = FileUtils.readLines(file, (String)null);
+ try {
+ Thread.sleep(500 * (i + 1));
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ i++;
+ }
+ assertEquals(lines.get(0), expectedString);
+
+ //empty the file
+ lines.clear();
+ FileUtils.writeLines(file, lines, false);
+ }
+
+ }
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
index bd3b4d0814,ca20dd89f7..1ae9ed91de
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
@@@ -211,10 -211,10 +211,10 @@@ public class TestPermissionCheck
bucketEndpoint.setClient(client);
try {
bucketEndpoint.get("bucketName", null, null, null, 1000,
- null, null, null, null, null, "acl", null);
+ null, null, null, null, "acl", null);
- } catch (Exception e) {
- Assert.assertTrue(e instanceof OS3Exception &&
- ((OS3Exception)e).getHttpCode() == HTTP_FORBIDDEN);
+ Assert.fail("Expected OS3Exception with FORBIDDEN http code.");
+ } catch (OS3Exception e) {
+ Assert.assertEquals(HTTP_FORBIDDEN, e.getHttpCode());
}
}
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
index 0000000000,ccf36a70c0..c8f421aae9
mode 000000,100644..100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
@@@ -1,0 -1,282 +1,284 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+ package org.apache.hadoop.ozone.s3.metrics;
+
+ import org.apache.commons.lang3.RandomStringUtils;
++import org.apache.hadoop.hdds.client.ReplicationConfig;
+ import org.apache.hadoop.hdds.client.ReplicationFactor;
+ import org.apache.hadoop.hdds.client.ReplicationType;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.ozone.OzoneConsts;
+ import org.apache.hadoop.ozone.client.OzoneBucket;
+ import org.apache.hadoop.ozone.client.OzoneClient;
+ import org.apache.hadoop.ozone.client.OzoneClientStub;
+ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+ import org.apache.hadoop.ozone.s3.endpoint.BucketEndpoint;
+ import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
+ import org.apache.hadoop.ozone.s3.endpoint.RootEndpoint;
+ import org.apache.hadoop.ozone.s3.endpoint.TestBucketAcl;
+ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+ import org.junit.Before;
+ import org.junit.Test;
+ import org.mockito.Mockito;
+
+ import javax.ws.rs.core.HttpHeaders;
+ import javax.ws.rs.core.Response;
+ import java.io.IOException;
+ import java.io.InputStream;
+ import java.util.HashMap;
+
+ import static java.net.HttpURLConnection.HTTP_OK;
+ import static java.nio.charset.StandardCharsets.UTF_8;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.fail;
+
+ /**
+ * Tests for {@link S3GatewayMetrics}.
+ */
+ public class TestS3GatewayMetrics {
+
+ private String bucketName = OzoneConsts.BUCKET;
+ private OzoneClient clientStub;
+ private BucketEndpoint bucketEndpoint;
+ private RootEndpoint rootEndpoint;
+ private ObjectEndpoint keyEndpoint;
+ private OzoneBucket bucket;
+ private HttpHeaders headers;
+ private static final String ACL_MARKER = "acl";
+ private S3GatewayMetrics metrics;
+
+
+ @Before
+ public void setup() throws Exception {
+ clientStub = new OzoneClientStub();
+ clientStub.getObjectStore().createS3Bucket(bucketName);
+ bucket = clientStub.getObjectStore().getS3Bucket(bucketName);
+
+ bucketEndpoint = new BucketEndpoint();
+ bucketEndpoint.setClient(clientStub);
+
+ rootEndpoint = new RootEndpoint();
+ rootEndpoint.setClient(clientStub);
+
+ keyEndpoint = new ObjectEndpoint();
+ keyEndpoint.setClient(clientStub);
+ keyEndpoint.setOzoneConfiguration(new OzoneConfiguration());
+
+ headers = Mockito.mock(HttpHeaders.class);
+ metrics = bucketEndpoint.getMetrics();
+ }
+
+ @Test
+ public void testHeadBucket() throws Exception {
+
+ long oriMetric = metrics.getHeadBucketSuccess();
+
+ bucketEndpoint.head(bucketName);
+
+ long curMetric = metrics.getHeadBucketSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testListBucket() throws Exception {
+
+ long oriMetric = metrics.getListS3BucketsSuccess();
+
+ rootEndpoint.get().getEntity();
+
+ long curMetric = metrics.getListS3BucketsSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testHeadObject() throws Exception {
+ String value = RandomStringUtils.randomAlphanumeric(32);
+ OzoneOutputStream out = bucket.createKey("key1",
- value.getBytes(UTF_8).length, ReplicationType.RATIS,
- ReplicationFactor.ONE, new HashMap<>());
++ value.getBytes(UTF_8).length,
++ ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
++ ReplicationFactor.ONE), new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+
+ long oriMetric = metrics.getHeadKeySuccess();
+
+ keyEndpoint.head(bucketName, "key1");
+
+ long curMetric = metrics.getHeadKeySuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetBucketSuccess() throws Exception {
+ long oriMetric = metrics.getGetBucketSuccess();
+
+ clientStub = createClientWithKeys("file1");
+ bucketEndpoint.setClient(clientStub);
+ bucketEndpoint.get(bucketName, null,
+ null, null, 1000, null,
+ null, "random", null,
+ null, null).getEntity();
+
+ long curMetric = metrics.getGetBucketSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetBucketFailure() throws Exception {
+ long oriMetric = metrics.getGetBucketFailure();
+
+ try {
+ // Searching for a bucket that does not exist
+ bucketEndpoint.get("newBucket", null,
+ null, null, 1000, null,
+ null, "random", null,
+ null, null);
+ fail();
+ } catch (OS3Exception e) {
+ }
+
+ long curMetric = metrics.getGetBucketFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testCreateBucketSuccess() throws Exception {
+
+ long oriMetric = metrics.getCreateBucketSuccess();
+
+ bucketEndpoint.put(bucketName, null,
+ null, null);
+ long curMetric = metrics.getCreateBucketSuccess();
+
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testCreateBucketFailure() throws Exception {
+ // Creating an error by trying to create a bucket that already exists
+ long oriMetric = metrics.getCreateBucketFailure();
+
+ bucketEndpoint.put(bucketName, null, null, null);
+
+ long curMetric = metrics.getCreateBucketFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testDeleteBucketSuccess() throws Exception {
+ long oriMetric = metrics.getDeleteBucketSuccess();
+
+ bucketEndpoint.delete(bucketName);
+
+ long curMetric = metrics.getDeleteBucketSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testDeleteBucketFailure() throws Exception {
+ long oriMetric = metrics.getDeleteBucketFailure();
+ bucketEndpoint.delete(bucketName);
+ try {
+ // Deleting a bucket that does not exist will result in delete failure
+ bucketEndpoint.delete(bucketName);
+ fail();
+ } catch (OS3Exception ex) {
+ assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), ex.getCode());
+ assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(),
+ ex.getErrorMessage());
+ }
+
+ long curMetric = metrics.getDeleteBucketFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetAclSuccess() throws Exception {
+ long oriMetric = metrics.getGetAclSuccess();
+
+ Response response =
+ bucketEndpoint.get(bucketName, null, null,
+ null, 0, null, null,
+ null, null, "acl", null);
+ long curMetric = metrics.getGetAclSuccess();
+ assertEquals(HTTP_OK, response.getStatus());
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetAclFailure() throws Exception {
+ long oriMetric = metrics.getGetAclFailure();
+ try {
+ // Failing the getACL endpoint by applying ACL on a non-Existent Bucket
+ bucketEndpoint.get("random_bucket", null,
+ null, null, 0, null,
+ null, null, null, "acl", null);
+ fail();
+ } catch (OS3Exception ex) {
+ assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), ex.getCode());
+ assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(),
+ ex.getErrorMessage());
+ }
+ long curMetric = metrics.getGetAclFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testPutAclSuccess() throws Exception {
+ long oriMetric = metrics.getPutAclSuccess();
+
+ clientStub.getObjectStore().createS3Bucket("b1");
+ InputStream inputBody = TestBucketAcl.class.getClassLoader()
+ .getResourceAsStream("userAccessControlList.xml");
+
+ bucketEndpoint.put("b1", ACL_MARKER, headers, inputBody);
+ inputBody.close();
+ long curMetric = metrics.getPutAclSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testPutAclFailure() throws Exception {
+ // Failing the putACL endpoint by applying ACL on a non-Existent Bucket
+ long oriMetric = metrics.getPutAclFailure();
+
+ InputStream inputBody = TestBucketAcl.class.getClassLoader()
+ .getResourceAsStream("userAccessControlList.xml");
+
+ try {
+ bucketEndpoint.put("unknown_bucket", ACL_MARKER, headers, inputBody);
+ fail();
+ } catch (OS3Exception ex) {
+ } finally {
+ inputBody.close();
+ }
+ long curMetric = metrics.getPutAclFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ private OzoneClient createClientWithKeys(String... keys) throws IOException {
+ OzoneBucket bkt = clientStub.getObjectStore().getS3Bucket(bucketName);
+ for (String key : keys) {
+ bkt.createKey(key, 0).close();
+ }
+ return clientStub;
+ }
+ }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org