You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/04/25 19:32:24 UTC
[ozone] branch HDDS-4440-s3-performance updated (55d2e06cf2 -> 2ea9e78bf1)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 55d2e06cf2 Fix checkstyle
add de0b1f9b81 HDDS-6528. Adding Unit-Test cases for S3-Gateway Bucket-Endpoint Metrics. (#3263)
add 4c1ba2b998 HDDS-6535. Ozone TokenRenewer not found (#3259)
add 141ced1196 HDDS-6579. Add commandline argument for Container Info command. (#3306)
add 22cdd95a9e HDDS-6141. terminate om if statemachine is shut down by ratis (#2953)
add c583b3e7c6 HDDS-6162. Limit OM DoubleBuffer pending requests to avoid taking too much memory (#3024)
add d8a21a2bdb HDDS-6593. Bump node to v16.14.2 for Recon (#3316)
add 0b4af02bfb HDDS-6591. Javadoc - update StorageContinerLocationProtocol for change of qualified class name (#3314)
add 2851ea53f0 HDDS-5339. Document OM bootstrap and decommission process (#3299)
add 8f98af911c HDDS-6547. om snapshot failed file not deleted. (#3268)
add fe49f69a33 HDDS-6346. OmFailoverProxyProvider cleanup in loadConfigs for legacy service ids code (#3313)
add d5eb702c68 HDDS-6587. SCM Web UI is not displaying HEALTHY_READONLY header in node status (#3311)
add a0684e26c8 HDDS-6574. Set owner of buckets created via S3 Gateway to actual user rather than `s3g`; print LinkBucket owner field on the client (#3298)
add 3fd7dc6c77 HDDS-6525. Add audit log for S3Gateway (#3252)
add 0b6f46714f HDDS-6562. Exclude specific operations in Audit log (#3289)
add 172947815b HDDS-6532. Enable READ audit log by default (#3255)
add 204584b395 HDDS-6515. Intermittent failure in TestRootedOzoneFileSystem (#3322)
add cec910568c HDDS-5003. Introduce EC replication type (#2064)
add a51efc541b HDDS-4954. Add replicaIndex to the RPC protocols (#2055)
add 23fe4f5e56 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add a915dbfdd5 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add c050b8f67e HDDS-5011. Introduce ECReplicationConfig and Java based ReplicationConfig implementation (#2068)
add aeedef8c7b Merge branch 'master' into the EC branch
add 1d5c593319 HDDS-5145. Extend Pipline/ReplicationConfig refactor with ECReplicationConfig (#2206)
add 132911c504 HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
add 9c4008d540 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add ad790b660e HDDS-4883. Persist replicationIndex on datanode side (#2069)
add c960ec7c5a Merge remote-tracking branch 'origin/HDDS-3816-ec' into HDDS-3816-ec
add f904539c0f Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add fbea53d4f5 Merge branch 'master' into EC branch
add b6f960daaa HDDS-5313. ContainerInfo should use ReplicationConfig (#2320)
add 9e0a2fc4e2 HDDS-5247. EC: Create ECReplicationConfig on client side based on input string (#2331)
add 5e42c31933 Merge branch 'master' into EC branch.
add 27f57b7d3c Fixed conflict issue as part of merge branch 'master' into EC branch.
add f53a93e6a9 HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
add 206c9358a5 HDDS-4892. EC: Implement basic EC pipeline provider (#2353)
add 3a94742765 HDDS-5375. EC: Extend PipelineManager.createPipeline API to support excluded nodes (#2378)
add 4115827ff9 HDDS-4940 : EC: Implement the ECKeyOutputStream which should handle the EC mode writes. (#2335)
add cd781b9785 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
add a0f6881a28 Merge branch 'master' into ec-merge-master
add 0c5c859b99 Fix issues caused by merging master into branch
add 2ef364f47b HDDS-5509. EC: Add missing break in switch statement when requesting EC blocks (#2468)
add a64945bd7c HDDS-5325. EC: Add configuration to set an EC container placement policy (#2483)
add 942a23f14e Merge remote-tracking branch 'origin/master' into merge-master-0803
add f3ce6c7801 HDDS-5510: allocateContainer should handle ec replication config (#2490)
add 6eba2ff17d HDDS-5543: OMKeyRequest#createFileInfo should handle ECReplicationConfig (#2501)
add 20851dc199 HDDS-5561. EC: ECReplicationConfig should be immutable (#2516)
add d2dca9b079 HDDS-5562. EC: ContainerPlacementPolicyFactory#getPolicyInternal() should not be public. (#2517)
add 94814e5dba HDDS-5425: Client side pipeline cache key should use host and port combination (#2510)
add 0e6e00e97e Merge branch 'master' into ec-branch
add db35f8c003 HDDS-4942. EC: Implement ECBlockInputStream to read a single EC BlockGroup (#2507)
add d6dc278195 EC: Make ECReplicationConfig stored as bucket level attributes. (#2401)
add 823b505ab5 HDDS-5326. EC: Create a new as many racks as possible placement policy for EC (#2497)
add 90c4d9286d HDDS-5470 : EC: Add padding and generate parity if the last stripe is not full (#2455)
add 2df6c9caf5 Merge branch 'master' into ec branch
add e08d5009ab Removed SCMPipelineManager.java which should have been removed via HDDS-5554, but was somehow missing on an earlier master to ec branch merge.
add 6b17c7570d HDDS-5745. EC: Pipeline builder should copy replica indexes from original pipeline (#2642)
add 1d3423de76 HDDS-5477. EC: commit key should consolidate and create one keyLocationInfo per blockGrp (#2648)
add ebdd1f5e3d HDDS-5682: EC: Provide replication config option from CLI when creating bucket. (#2639)
add b55107fe76 HDDS-5550. EC: Adapt KeyInputStream to read EC Blocks (#2653)
add 0a0e62e002 EC: Add Codec and Stripe Size to ECReplicationConfig (#2674)
add fc17518932 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
add 5cf251654e HDDS-5741. EC: Remove hard coded chunksize and get from from ReplicationConfig (#2687)
add 831b3faa27 HDDS-5833: EC: Fix TestRootedOzoneFileSystem.testBucketDefaultsShouldBeInheritedToFileForEC failure in branch (#2718)
add 7a67f63429 HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
add a16ee9d2fb Merge branch 'master' into HDDS-3816-ec
add aee577c171 HDDS-5825: EC: ECKeyOutputStream#close fails if we write the partial chunk (#2720)
add f478b96360 HDDS-5832. EC: ECKeyOutputStream persists blocks in random order (#2717)
add d37fe6901b HDDS-5552. EC: Implement seek on ECBlockInputStream (#2723)
add 38e4cca273 HDDS-5755 EC: Refactor ECBlockOutputStreamEntry to accommodate all block group related ECBlockOuputStreams. (#2702)
add 914ce67af3 HDDS-5364: EC: Adopt EC related utility from Hadoop source repository (#2733)
add f5dbe00309 Hdds 5491: EC: Write should handle node failures. (#2767)
add c9fa9bdf0f HDDS-5874 EC: Integrate the Codec changes into EC Streams. (#2777)
add 818118a7ad HDDS-5948: EC: Fix the compile issue in TestOzoneECClient (Due to concurrent commits) (#2812)
add e82682fe8b HDDS-5551. EC: Implement an Input Stream to reconstruct EC blocks on demand (#2797)
add f85ac376e6 HDDS-5950. EC: Implement seek on ECBlockReconstructedStripeInputStream (#2818)
add 438799ba75 HDDS-5953: EC: Review the current flush API and clean up (#2828)
add 52f05fc349 HDDS-5951. EC: ECBlockReconstructedStripeInputStream should handle block read failures and continue reading (#2831)
add 67043109c0 Merging master into EC branch
add d5ac6738b7 HDDS-6012: EC: Fix TestOzoneShellHA failures post master merge with EC branch
add 9a7025a39b HDDS-6035: EC: Fix TestOmMetrics in merge branch
add a0349bd9a9 HDDS-6029: EC: Fix the replication config handling in OMDirectoryCreateRequest#dirKeyInfoBuilderNoACL (#1)
add 685442c6b8 HDDS-6001. EC: Create ECBlockReconstructedInputStream to wrap ECBlockReconstructedStripeInputStream (#2848)
add 93fadc9789 HDDS-5979: EC: Change CLI bucket default replication option name to -type (#2873)
add 6e7d0dc206 HDDS-6009. EC: Optimize ECBlockReconstructedStripeInputStream where there are no missing data indexes (#2872)
add 1b9724d60a HDDS-5955: EC: Track the failed servers to add into the excludeList when invoking allocateBlock (#2849)
add 460aa9b89b HDDS-6020: EC: Provide set replicationConfig option to bucket (#2880)
add a1d7292bcf HDDS-6010. EC: Create ECBlockInputStreamProxy to choose between reconstruction and normal reads (#2889)
add 73bddf36d0 Updates to address compile issues after merge
add 3007b5c68d HDDS-6081. EC: Fix ReplicationConfig related test failures that happened due to merging HDDS-5997 to the EC branch
add c6604833d1 HDDS-5952. EC: ECBlockReconstructedStripeInputStream should read from blocks in parallel (#2899)
add 86f735418a HDDS-6036: EC: handleStripeFailure should retry (#2910)
add f8fd2ff66b HDDS-6065: EC: Provide CLI option to reset the bucket replication config (#2927)
add ba192b1de1 HDDS-6132: EC: HandleStripeFailure should not release the cachebuffers. (#2940)
add 3eb55db014 HDDS-5949. EC: Create reusable buffer pool shared by all EC input and output streams (#2929)
add 5856411168 HDDS-6179. EC: Provide toString in ECReplicationConfig (#2986)
add 07b2338525 HDDS-6180. EC: Pipeline creator should ignore creating pipelines for ZERO factor (#2985)
add aaf6f17045 HDDS-6021: EC: Client side exclude nodes list should expire after certain time period or based on the list size. (#2973)
add d0467bac1d HDDS-6220. EC: Introduce a gRPC client implementation for EC with really async WriteChunk and PutBlock. (#3016)
add b40ab3f97b HDDS-6172: EC: Document the Ozone EC (#3006)
add 78ccdd540c HDDS-6131: EC: Replication config from bucket should be refreshed in o3fs (#3008)
add 6b5c01c843 HDDS-6184. EC: put command should create EC key if bucket is EC (#2990)
add 4d660c4d79 HDDS-6233. EC: Bucket does not display correct EC replication details (#3025)
add 2e60a4f8cb Merge master to EC Branch
add 58f8e30538 HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist (#3029)
add 78ca097eac HDDS-6083: EC: Fix flakyness of tests around nodefailures (#3021)
add dfa8141937 HDDS-6237: EC: putBlock should pass close flag true on end of block group/close file. (#3026)
add ee630e2995 HDDS-6240. EC: Container Info command with json switch fails for EC containers (#3033)
add 8e52561eab HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3037)
add ddcfc521af HDDS-6252. EC: Datanode Chunk Validator fails on encountering EC pipeline (#3035)
add ca6e4c3599 HDDS-6255. EC: Replication Manager should skip EC Containers (#3038)
add 5d2fa94949 HDDS-6256. EC: Parity blocks are incorrectly padded with zeros to the chunk size (#3043)
add 9a0f812b3b HDDS-6231. EC: Create EC acceptance test environment and some basic tests (#3030)
add 722c444aea HDDS-6250. EC: Add replica index to the output in the container info command (#3041)
add 29c69b109b HDDS-6272: EC: Recon UI failing because of EC replication factor (#3067)
add 22e34820cf HDDS-6258. EC: Read with stopped but not dead nodes gives IllegalStateException rather than InsufficientNodesException (#3048)
add c4aec7bb97 HDDS-6259. EC: Pipelines for closed containers should contain correct replica indexes (#3062)
add dafdb23255 HDDS-6295. EC: Fix unaligned stripe write failure due to length overflow. (#3076)
add 2d63cc4ec9 HDDS-6194. EC: Freon ockg support EC write (#2996)
add 79f60c4d5c HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC libraries (#3087)
add aebaff55a2 Merge master to EC branch HDDS-3816-ec
add b2a33b759e HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
add 4b9d38763a HDDS-6294. EC: Make cluster-wide EC configuration take effect (#3089)
add 550eadc970 HDDS-6308. EC: Handle Replication Factor to consider EC Config in Recon UI (#3078)
add 6f64eb77d5 HDDS-6342. EC: Fix large write with multiple stripes upon stripe failure. (#3108)
add 60a41bc85e HDDS-6249. EC: Fix todo items in TestECKeyOutputStream (#3102)
add 1d23619164 HDDS-6188: EC: Adapt java side native coder classes from hadoop. (#3046)
add 387fb1a2c1 HDDS-6232. EC: Update help strings for replication config (#3113)
add 0a83e7018e HDDS-6341: EC: Fix the race condition in TestECBlockReconstructedStripeInputStream. (#3111)
add aac07ef790 HDDS-6347. EC: Freon randomKeys EC key support (#3112)
add 6b8d935f73 HDDS-6319. EC: Fix read big file failure with EC policy 10+4. (#3094)
add b9adbcd89a HDDS-6348: EC: PartialStripe failure handling logic is writing padding bytes also to DNs (#3124)
add b90f00192d HDDS-6372. EC: Do not throw NotImplementedException in flush() (#3133)
add 78ca327e77 HDDS-6358. EC: Refactor ECKeyOutputStream#write() (#3120)
add 9e3cf98a30 HDDS-6279. EC: Calculate EC replication correctly when updating bucket usage (#3136)
add e675e3ba3d HDDS-6380. EC: Key Info command should not display legacy replication fields as they duplicate ReplicationConfig (#3141)
add 9ee6d72f31 HDDS-6373. EC: Exclude pipeline upon container close instead of exclude DNs. (#3137)
add 38409ea5a0 HDDS-6364. EC: Discard pre-allocated blocks to eliminate worthless retries. (#3127)
add 9680edf850 HDDS-6384. EC: Ensure EC container usage is updated correctly when handling reports (#3147)
add 58eac935cd HDDS-6185: EC: EC keys can't be created via S3 interfaces (#3150)
add 8cff9fd2ad HDDS-6368. EC: Fix broken future chain and cleanup unnecessary validation function. (#3128)
add 8776430812 HDDS-6382. EC: ListPipelines command should consider EC Containers (#3161)
add 28482bf2b9 HDDS-6422. EC: Fix too many idle threads during reconstruct read. (#3168)
add 074203fc0a HDDS-6437. EC: Avoid allocating buffers in EC Reconstruction Streams until first read (#3181)
add 9675863705 HDDS-6413. EC: OzoneManagerRequestHandler needs to handle ECReplicationConfig (#3198)
add 06f1eb5ad6 HDDS-6248. EC: Container list command should allow filtering of EC containers (#3179)
add 03ff3f9592 HDDS-6445. EC: Fix allocateBlock failure due to inaccurate excludedNodes check. (#3190)
add af48c537a9 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
add 217c1911b2 EC: Check isFullCell inside handleDataWrite (#3202)
add 9cda6a1a06 HDDS-6460. EC: Adjust requested size by EC DataNum in WritableECContainerProvider (#3203)
add 04c62c53f2 HDDS-6411. EC: OmMultipartKeyInfo needs to handle ECReplicationConfig (#3210)
add fc6ac0c48c HDDS-6448. EC: Reconstructed Input Streams should free resources after reading to end of block (#3197)
add f386662a37 HDDS-6381. EC: Improve exception message in ByteBufferEncodingState (#3143)
add 9ec90f5820 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec-merge-master
add c55f30edf0 HDDS-6412: EC: Handle ECReplicationConfig in initiateMultipartUpload API (#3184)
add 16c3e99eae HDDS-6478. EC: Fix ISA-l load hadoop native lib UnsatisfiedLinkError (#3218)
add 143315c89b HDDS-6409. EC: OzoneMultipartUpload needs to handle ECReplicationConfig (#3279)
add d9529b32ea HDDS-6434: EC: Overwriting an EC key with a Ratis key fails (#3274)
add df96b86c8d HDDS-6557. EC: Execute S3 acceptance tests with EC (#3281)
add e9b6215210 Merge master to EC Branch HDDS-3816-ec
add e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
add 6c6b740a47 HDDS-6209. EC: [Forward compatibility issue] New client to older server could fail due to the unavailability for client default replication config (#3267)
add 70b547be23 HDDS-5909. EC: Onboard EC into upgrade framework (#3262)
add cb0a84b27c Merge master to EC branch HDDS-3816-ec
add 96669856ad Merge EC branch HDDS-3816-ec to master
add 681d0a8f5e HDDS-6471. Only repeat flaky tests (#3247)
add dc9da7d405 HDDS-6606. Rat check failure ignored for ozone-annotation-processing (#3321)
add be5888b907 HDDS-6580. Introduce KEY_PATH_LOCK under OzoneManagerLock.Resource (#3308)
add 4345fb1c58 HDDS-6189. Intermittent failure in TestDirectoryDeletingServiceWithFSO (#3328)
add 03b0a61958 HDDS-6583. Introduce lock strategy pattern implementations based on configuration flags (#3310)
add cb2754fd9c HDDS-6555. Container Deletion should not depend on usedBytes being zero (#3276)
add b077cc6123 HDDS-6561. Modify default bucket layout to LEGACY. (#3296)
add 9ac8fbce70 HDDS-6614. EC: Fix Datanode block file INCONSISTENCY during heavy load. (#3323)
add 2876c54b8d HDDS-6405. Selective checks: run rat for readme change (#3156)
add 497c7a40b8 HDDS-6617. Skip compile(11) for non-java changes (#3324)
add b8388b096a HDDS-6639. Remove flaky tag for testOMProxyProviderFailoverOnConnectionFailure (#3343)
add b78adc4c71 HDDS-6620. Gather AuditLog params for S3g (#3325)
add de9fcca1c3 HDDS-6522. Fix TestOzoneManagerHAWithData#testTwoOMNodesDown (#3341)
new 2ea9e78bf1 Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 50 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../hdds/scm/storage/BlockExtendedInputStream.java | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 177 ++++
.../hdds/scm/storage/ExtendedInputStream.java | 91 ++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 134 +++
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 16 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 29 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../common/src/main/resources/ozone-default.xml | 117 ++-
.../hdds/client/TestECReplicationConfig.java | 91 ++
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +++-
.../hdds/scm/container/TestContainerInfo.java | 86 ++
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/common/helpers/TestExcludeList.java | 67 ++
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 ++
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 54 +-
.../common/src/test/resources/auditlog.properties | 2 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++++
hadoop-hdds/docs/content/feature/OM-HA.md | 37 +
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 ++
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 ++
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +++
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 ++
.../rawcoder/AbstractNativeRawEncoder.java | 99 ++
.../rawcoder/ByteArrayDecodingState.java | 132 +++
.../rawcoder/ByteArrayEncodingState.java | 105 ++
.../rawcoder/ByteBufferDecodingState.java | 143 +++
.../rawcoder/ByteBufferEncodingState.java | 110 ++
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 ++
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 ++
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 ++
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 ++
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 ++
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 ++
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 ++
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 ++
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 ++
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 ++
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 ++++++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++++++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 ++
.../ozone/erasurecode/TestCodecRegistry.java | 159 +++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++++++++
.../apache/ozone/erasurecode}/package-info.java | 12 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 ++
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 ++
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +++
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 ++
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 1 +
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 20 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ReplicationManager.java | 20 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 ++++++
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 10 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +++
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 32 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 30 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 ++++-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 96 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../scm/container/TestContainerManagerImpl.java | 10 +
.../scm/container/TestContainerReportHandler.java | 181 +++-
.../hdds/scm/container/TestReplicationManager.java | 72 +-
.../algorithms/TestContainerPlacementFactory.java | 8 +
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +++---
.../hdds/scm/ha/TestReplicationAnnotation.java | 5 +
.../hdds/scm/pipeline/MockPipelineManager.java | 54 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 +++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 49 +
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 61 ++
.../pipeline/TestWritableECContainerProvider.java | 446 ++++++++
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 37 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 75 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 192 ++++
hadoop-ozone/client/pom.xml | 4 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 ++
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 420 ++++++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 ++
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 ++
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 +++++
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 +++++++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 +++++
.../io/ECBlockReconstructedStripeInputStream.java | 631 ++++++++++++
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 706 +++++++++++++
.../client/io/InsufficientLocationsException.java | 26 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 233 ++++-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 ++-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 +++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1066 ++++++++++++++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 28 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 ++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 9 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 58 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 ++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 41 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 ++
.../dist/src/main/smoketest/ec/basic.robot | 64 ++
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 8 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 2 +-
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
...g4j2.properties => s3g-audit-log4j2.properties} | 8 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 70 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 235 +++--
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 435 +++++---
.../ozone/client/rpc/read/ECStreamTestUtil.java | 408 ++++++++
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 111 ++
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../client/rpc/read/TestECBlockInputStream.java | 522 ++++++++++
.../rpc/read/TestECBlockInputStreamProxy.java | 399 ++++++++
.../read/TestECBlockReconstructedInputStream.java | 379 +++++++
.../TestECBlockReconstructedStripeInputStream.java | 724 +++++++++++++
.../ozone/client/rpc/read/TestInputStreamBase.java | 23 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../client/rpc/read/TestKeyInputStreamEC.java | 118 +++
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 ++--
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 3 +
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 -
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 156 ++-
.../src/main/proto/OmClientProtocol.proto | 20 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 66 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 ++
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 ++
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/request/bucket/OMBucketCreateRequest.java | 28 +
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 63 +-
.../request/file/OMFileCreateRequestWithFSO.java | 15 +-
.../om/request/key/OMAllocateBlockRequest.java | 37 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 31 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 58 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 30 +-
.../S3MultipartUploadCompleteRequest.java | 41 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 5 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 11 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 ++++-
.../hadoop/ozone/om/TestKeyDeletingService.java | 7 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 ++
.../apache/hadoop/ozone/om/TestTrashService.java | 2 +
.../ozone/om/lock/TestOzoneLockProvider.java | 108 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../request/validation/TestRequestValidations.java | 29 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +++
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +++-
.../org.apache.hadoop.security.token.DtFetcher | 0
...rg.apache.hadoop.security.token.TokenIdentifier | 0
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 5 +
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../scm/ReconStorageContainerManagerFacade.java | 7 +
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
hadoop-ozone/s3gateway/pom.xml | 4 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 ++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 190 ++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 75 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 183 ++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 53 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 52 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 75 ++
.../hadoop/ozone/client/OzoneBucketStub.java | 69 +-
...ewayMetrics.java => TestS3GatewayAuditLog.java} | 102 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 5 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 173 +++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 +++
.../src/test/resources/auditlog.properties | 4 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 109 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 6 +
411 files changed, 26425 insertions(+), 2043 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ECXceiverClientGrpc.java
copy hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java => hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockExtendedInputStream.java (63%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ExtendedInputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/{AuditLoggerType.java => S3GAction.java} (60%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestECReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerInfo.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/common/helpers/TestExcludeList.java
create mode 100644 hadoop-hdds/docs/content/feature/EC-Chunk-Layout.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reads-With-No-Failures.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reconstructional-Read.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Write-Block-Allocation-in-Containers.png
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.md
copy {hadoop-ozone/client => hadoop-hdds/erasurecode}/pom.xml (57%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/HadoopNativeECAccessorUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder}/package-info.java (73%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/BufferAllocator.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/CodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/ECChunk.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode}/package-info.java (69%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/CoderUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/EncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ErasureCodeNative.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/package-info.java} (50%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/CodecUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GF256.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GaloisField.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/RSUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util}/package-info.java (74%)
create mode 100644 hadoop-hdds/erasurecode/src/main/resources/META-INF/services/org.apache.ozone.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode}/package-info.java (83%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderBenchmark.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestCodecRawCoderMapping.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestDummyRawCoder.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeRSRawCoder.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java} (57%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoder.java} (69%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoderBase.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawCoderBase.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java (51%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoder.java} (75%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder}/package-info.java (83%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
copy hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/{TestSCMContainerPlacementRackAware.java => TestSCMContainerPlacementRackScatter.java} (70%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateMap.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/pipeline/TestListPipelinesSubCommand.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntryPool.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
copy hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java (58%)
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/BlockStreamAccessor.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestQuotaUtil.java
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
create mode 100644 hadoop-ozone/dist/src/main/org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot
copy hadoop-ozone/dist/src/shell/conf/{scm-audit-log4j2.properties => s3g-audit-log4j2.properties} (94%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/ECStreamTestUtil.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStreamProxy.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockReconstructedInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStreamEC.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OBSKeyPathLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/RegularBucketLockStrategy.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOzoneLockProvider.java
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (92%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
copy hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/{metrics/TestS3GatewayMetrics.java => TestS3GatewayAuditLog.java} (53%)
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
copy {hadoop-hdds/common => hadoop-ozone/s3gateway}/src/test/resources/auditlog.properties (97%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
copy hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java => tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java} (62%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2ea9e78bf1bd041a140a18c5bf7c0110a7fcc570
Merge: 55d2e06cf2 de9fcca1c3
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Mon Apr 25 18:39:39 2022 +0200
Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
.github/workflows/post-commit.yml | 50 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../hdds/scm/storage/BlockExtendedInputStream.java | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 177 ++++
.../hdds/scm/storage/ExtendedInputStream.java | 91 ++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 134 +++
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 16 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 29 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../common/src/main/resources/ozone-default.xml | 117 ++-
.../hdds/client/TestECReplicationConfig.java | 91 ++
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +++-
.../hdds/scm/container/TestContainerInfo.java | 86 ++
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/common/helpers/TestExcludeList.java | 67 ++
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 ++
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 54 +-
.../common/src/test/resources/auditlog.properties | 2 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++++
hadoop-hdds/docs/content/feature/OM-HA.md | 37 +
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 ++
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 ++
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +++
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 ++
.../rawcoder/AbstractNativeRawEncoder.java | 99 ++
.../rawcoder/ByteArrayDecodingState.java | 132 +++
.../rawcoder/ByteArrayEncodingState.java | 105 ++
.../rawcoder/ByteBufferDecodingState.java | 143 +++
.../rawcoder/ByteBufferEncodingState.java | 110 ++
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 ++
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 ++
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 ++
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 ++
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 ++
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 ++
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 ++
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 ++
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 ++
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 ++
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 ++++++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++++++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 ++
.../ozone/erasurecode/TestCodecRegistry.java | 159 +++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++++++++
.../apache/ozone/erasurecode}/package-info.java | 12 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 ++
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 ++
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +++
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 ++
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 1 +
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 20 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ReplicationManager.java | 20 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 ++++++
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 10 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +++
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 32 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 30 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 ++++-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 96 +-
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../scm/container/TestContainerManagerImpl.java | 10 +
.../scm/container/TestContainerReportHandler.java | 181 +++-
.../hdds/scm/container/TestReplicationManager.java | 72 +-
.../algorithms/TestContainerPlacementFactory.java | 8 +
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +++---
.../hdds/scm/ha/TestReplicationAnnotation.java | 5 +
.../hdds/scm/pipeline/MockPipelineManager.java | 54 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 +++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 49 +
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 61 ++
.../pipeline/TestWritableECContainerProvider.java | 446 ++++++++
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 37 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 75 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 192 ++++
hadoop-ozone/client/pom.xml | 4 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 ++
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 420 ++++++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 ++
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 ++
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 +++++
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 +++++++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 +++++
.../io/ECBlockReconstructedStripeInputStream.java | 631 ++++++++++++
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 706 +++++++++++++
.../client/io/InsufficientLocationsException.java | 26 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 233 ++++-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 ++-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 +++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1066 ++++++++++++++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 28 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 ++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 9 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 58 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 ++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 41 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 ++
.../dist/src/main/smoketest/ec/basic.robot | 64 ++
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 8 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 2 +-
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
...g4j2.properties => s3g-audit-log4j2.properties} | 8 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 70 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 235 +++--
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 435 +++++---
.../ozone/client/rpc/read/ECStreamTestUtil.java | 408 ++++++++
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 111 ++
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../client/rpc/read/TestECBlockInputStream.java | 522 ++++++++++
.../rpc/read/TestECBlockInputStreamProxy.java | 399 ++++++++
.../read/TestECBlockReconstructedInputStream.java | 379 +++++++
.../TestECBlockReconstructedStripeInputStream.java | 724 +++++++++++++
.../ozone/client/rpc/read/TestInputStreamBase.java | 23 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../client/rpc/read/TestKeyInputStreamEC.java | 118 +++
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 ++--
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 3 +
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 -
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 156 ++-
.../src/main/proto/OmClientProtocol.proto | 20 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 66 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 ++
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 ++
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/request/bucket/OMBucketCreateRequest.java | 28 +
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 63 +-
.../request/file/OMFileCreateRequestWithFSO.java | 15 +-
.../om/request/key/OMAllocateBlockRequest.java | 37 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 31 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 58 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 30 +-
.../S3MultipartUploadCompleteRequest.java | 41 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 5 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 11 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 ++++-
.../hadoop/ozone/om/TestKeyDeletingService.java | 7 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 ++
.../apache/hadoop/ozone/om/TestTrashService.java | 2 +
.../ozone/om/lock/TestOzoneLockProvider.java | 108 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../request/validation/TestRequestValidations.java | 29 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +++
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +++-
.../org.apache.hadoop.security.token.DtFetcher} | 2 +-
...g.apache.hadoop.security.token.TokenIdentifier} | 6 +-
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 5 +
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../scm/ReconStorageContainerManagerFacade.java | 7 +
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
hadoop-ozone/s3gateway/pom.xml | 4 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 ++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 190 ++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 75 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 183 ++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 53 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 52 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 75 ++
.../hadoop/ozone/client/OzoneBucketStub.java | 69 +-
...ewayMetrics.java => TestS3GatewayAuditLog.java} | 102 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 5 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 173 +++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 +++
.../src/test/resources/auditlog.properties | 4 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 109 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 6 +
411 files changed, 26428 insertions(+), 2048 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index cb3b3e6fd9,436d5ab39a..6c76ca6156
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -3039,17 -3096,10 +3095,17 @@@
OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds.
</description>
</property>
-
+ <property>
+ <name>ozone.om.grpc.maximum.response.length</name>
+ <value>134217728</value>
+ <tag>OZONE, OM, S3GATEWAY</tag>
+ <description>
+ OM/S3GATEWAY OMRequest, OMResponse over grpc max message length (bytes).
+ </description>
+ </property>
<property>
<name>ozone.default.bucket.layout</name>
- <value>OBJECT_STORE</value>
+ <value/>
<tag>OZONE, MANAGEMENT</tag>
<description>
Default bucket layout used by Ozone Manager during bucket creation when a client does not specify the
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index e28907771d,c8af3e6e4b..bc1ed8d6f5
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -317,9 -320,17 +331,21 @@@ public final class OMConfigKeys
"ozone.om.admin.protocol.wait.between.retries";
public static final long OZONE_OM_ADMIN_PROTOCOL_WAIT_BETWEEN_RETRIES_DEFAULT
= 1000;
-
+ public static final String OZONE_OM_TRANSPORT_CLASS =
+ "ozone.om.transport.class";
+ public static final String OZONE_OM_TRANSPORT_CLASS_DEFAULT =
+ "org.apache.hadoop.ozone.om.protocolPB"
+ + ".Hadoop3OmTransportFactory";
+ public static final String OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT =
+ "ozone.om.unflushed.transaction.max.count";
+ public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
+ = 10000;
+
+ /**
+ * This configuration shall be enabled to utilize the functionality of the
+ * fine-grained KEY_PATH_LOCK.
+ */
+ public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED =
+ "ozone.om.key.path.lock.enabled";
+ public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false;
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
index 37f6b84f3d,001322e2af..985b3e76ba
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
@@@ -133,38 -133,36 +133,34 @@@ public class OMFailoverProxyProvider<T
this.omProxyInfos = new HashMap<>();
this.omNodeIDList = new ArrayList<>();
- Collection<String> omServiceIds = Collections.singletonList(omSvcId);
+ Collection<String> omNodeIds = OmUtils.getActiveOMNodeIds(config,
+ omSvcId);
- for (String serviceId : OmUtils.emptyAsSingletonNull(omServiceIds)) {
- Collection<String> omNodeIds = OmUtils.getActiveOMNodeIds(config,
- serviceId);
+ for (String nodeId : OmUtils.emptyAsSingletonNull(omNodeIds)) {
- for (String nodeId : OmUtils.emptyAsSingletonNull(omNodeIds)) {
+ String rpcAddrKey = ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
+ omSvcId, nodeId);
+ String rpcAddrStr = OmUtils.getOmRpcAddress(config, rpcAddrKey);
+ if (rpcAddrStr == null) {
+ continue;
+ }
- String rpcAddrKey = ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
- serviceId, nodeId);
- String rpcAddrStr = OmUtils.getOmRpcAddress(config, rpcAddrKey);
- if (rpcAddrStr == null) {
- continue;
- }
+ OMProxyInfo omProxyInfo = new OMProxyInfo(omSvcId, nodeId,
+ rpcAddrStr);
- OMProxyInfo omProxyInfo = new OMProxyInfo(serviceId, nodeId,
- rpcAddrStr);
-
- if (omProxyInfo.getAddress() != null) {
- // For a non-HA OM setup, nodeId might be null. If so, we assign it
- // the default value
- if (nodeId == null) {
- nodeId = OzoneConsts.OM_DEFAULT_NODE_ID;
- }
- // ProxyInfo will be set during first time call to server.
- omProxies.put(nodeId, null);
- omProxyInfos.put(nodeId, omProxyInfo);
- omNodeIDList.add(nodeId);
- } else {
- LOG.error("Failed to create OM proxy for {} at address {}",
- nodeId, rpcAddrStr);
+ if (omProxyInfo.getAddress() != null) {
-
-
+ // For a non-HA OM setup, nodeId might be null. If so, we assign it
+ // the default value
+ if (nodeId == null) {
+ nodeId = OzoneConsts.OM_DEFAULT_NODE_ID;
}
+ // ProxyInfo will be set during first time call to server.
+ omProxies.put(nodeId, null);
+ omProxyInfos.put(nodeId, omProxyInfo);
+ omNodeIDList.add(nodeId);
+ } else {
+ LOG.error("Failed to create OM proxy for {} at address {}",
+ nodeId, rpcAddrStr);
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index fa33d81949,b23962169c..a40461dcb3
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -64,10 -66,13 +65,15 @@@ public class TestOzoneConfigurationFiel
xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms");
xmlPropsToSkipCompare.add("ozone.om.leader.election.minimum.timeout" +
".duration"); // Deprecated config
+ xmlPrefixToSkipCompare.add("ozone.s3g");
+ configurationPrefixToSkipCompare.add("ozone.s3g");
configurationPropsToSkipCompare
.add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION);
+ // Currently replication and type configs moved to server side.
+ configurationPropsToSkipCompare
+ .add(OzoneConfigKeys.OZONE_REPLICATION);
+ configurationPropsToSkipCompare
+ .add(OzoneConfigKeys.OZONE_REPLICATION_TYPE);
configurationPropsToSkipCompare
.add(OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY);
configurationPropsToSkipCompare
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 1e9e343034,9a0efc7d15..b08bfdb91f
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@@ -131,14 -137,9 +149,15 @@@ public abstract class EndpointBase impl
try {
client.getObjectStore().createS3Bucket(bucketName);
} catch (OMException ex) {
+ getMetrics().incCreateBucketFailure();
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
+ } else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
+ throw newError(S3ErrorTable.ACCESS_DENIED,
+ s3Auth.getAccessID(), ex);
+ } else if (ex.getResult() == ResultCodes.TIMEOUT ||
+ ex.getResult() == ResultCodes.INTERNAL_ERROR) {
+ throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
} else if (ex.getResult() != ResultCodes.BUCKET_ALREADY_EXISTS) {
// S3 does not return error for bucket already exists, it just
// returns the location.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org