You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/05/24 07:34:29 UTC
[ozone] branch HDDS-4944 updated (35043683a6 -> 79a812af1f)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 35043683a6 HDDS-6371. [Multi-Tenant] Provide OM DB to Apache Ranger Sync mechanism (#3131)
add cf8eb336ec HDDS-5210. Create GRPC service definition for existing OM protocol (#2375)
add 3b4e2f58b3 Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
add 1bce6dfd11 HDDS-5213. Create and a specific OmTransportFactory for GrpcOmTransport (#2485)
add 46575efe21 HDDS-5211. Start OM Grpc server as part of the OM bootstrap (#2601)
add a115ebfadc HDDS-5212. Create implementation of OmTransport which uses Grpc (#2580)
add 1b891fb64f HDDS-5612. Create Ozone s3 authentication token and client authentication transport (#2655)
add b0f41d67da HDDS-5780. Support s3 authentication on a per request basis (#2721)
add 1ffbeddcec HDDS 5781. Enable ACLs and support for all s3 file operations. (#2739)
add c9442d2507 HDDS-5778. Client side unit tests (#2810)
add 4d0bf19895 Commit for merge requested in PR #2945 to merge Apache Ozone master branch with HDDS-5881 changes INTO HDDS-4400-s3-performance feature branch. Merge allows s3 gateway gRPC to work with s3 gateway persistent connection changes made in HDDS-5881 found in master branch.
add abb3637a08 Synchronization added to getOzoneConfiguration for OzoneClientProducer to satisfy CI findbugs.
add b88a45e994 HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTransport support (#3074)
add 413d4aade2 HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)
add 0b0c586fec HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)
add 3f585ab05d HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)
add d5ade578f4 Revert "HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)"
add d87a7461c1 Revert "HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)"
add d57616e165 Revert "HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)"
add 2522127b1c Revert "HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTransport support (#3074)"
add f7b2dbd8f2 Necessary due to CI workflow issue, HDDS-6239, impacting feature branch green builds. Merging with Master branch impacts HDDS-4440 PRs in progress. This merge also requires partial changes that were initially in PR for HDDS-5545 to be included. Namely, the patch to allow switching between OmTransports, GrpcOmTransport and Hadoop3OmTransport through ozone configuration in the OmConfigKeys.
add e3259164ca Trigger Build
add 666525e386 Fix integration test for added configuation field for selecting OmTransport for s3 gateway - TestOzoneConfigurationFields (added config key not in xml).
add 0de099dbde Merge remote-tracking branch 'neils-dev/merge-master-s3g' into HDDS-4440-s3-performance
add 76e238118d HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)
add 47e457697e HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)
add 062890ee4b HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)
add a2e169393c Merge of master into feature branch to sync the feature branch to the current master. Resolved conflicts affecting feature versioning PR#3155, propagating s3 exception stack trace HDDS-6257 PR#3066 and performFailover PR#3160.
add 55d2e06cf2 Fix checkstyle
add 172947815b HDDS-6532. Enable READ audit log by default (#3255)
add 204584b395 HDDS-6515. Intermittent failure in TestRootedOzoneFileSystem (#3322)
add cec910568c HDDS-5003. Introduce EC replication type (#2064)
add a51efc541b HDDS-4954. Add replicaIndex to the RPC protocols (#2055)
add 23fe4f5e56 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add a915dbfdd5 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add c050b8f67e HDDS-5011. Introduce ECReplicationConfig and Java based ReplicationConfig implementation (#2068)
add aeedef8c7b Merge branch 'master' into the EC branch
add 1d5c593319 HDDS-5145. Extend Pipline/ReplicationConfig refactor with ECReplicationConfig (#2206)
add 132911c504 HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
add 9c4008d540 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add ad790b660e HDDS-4883. Persist replicationIndex on datanode side (#2069)
add c960ec7c5a Merge remote-tracking branch 'origin/HDDS-3816-ec' into HDDS-3816-ec
add f904539c0f Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add fbea53d4f5 Merge branch 'master' into EC branch
add b6f960daaa HDDS-5313. ContainerInfo should use ReplicationConfig (#2320)
add 9e0a2fc4e2 HDDS-5247. EC: Create ECReplicationConfig on client side based on input string (#2331)
add 5e42c31933 Merge branch 'master' into EC branch.
add 27f57b7d3c Fixed conflict issue as part of merge branch 'master' into EC branch.
add f53a93e6a9 HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
add 206c9358a5 HDDS-4892. EC: Implement basic EC pipeline provider (#2353)
add 3a94742765 HDDS-5375. EC: Extend PipelineManager.createPipeline API to support excluded nodes (#2378)
add 4115827ff9 HDDS-4940 : EC: Implement the ECKeyOutputStream which should handle the EC mode writes. (#2335)
add cd781b9785 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
add a0f6881a28 Merge branch 'master' into ec-merge-master
add 0c5c859b99 Fix issues caused by merging master into branch
add 2ef364f47b HDDS-5509. EC: Add missing break in switch statement when requesting EC blocks (#2468)
add a64945bd7c HDDS-5325. EC: Add configuration to set an EC container placement policy (#2483)
add 942a23f14e Merge remote-tracking branch 'origin/master' into merge-master-0803
add f3ce6c7801 HDDS-5510: allocateContainer should handle ec replication config (#2490)
add 6eba2ff17d HDDS-5543: OMKeyRequest#createFileInfo should handle ECReplicationConfig (#2501)
add 20851dc199 HDDS-5561. EC: ECReplicationConfig should be immutable (#2516)
add d2dca9b079 HDDS-5562. EC: ContainerPlacementPolicyFactory#getPolicyInternal() should not be public. (#2517)
add 94814e5dba HDDS-5425: Client side pipeline cache key should use host and port combination (#2510)
add 0e6e00e97e Merge branch 'master' into ec-branch
add db35f8c003 HDDS-4942. EC: Implement ECBlockInputStream to read a single EC BlockGroup (#2507)
add d6dc278195 EC: Make ECReplicationConfig stored as bucket level attributes. (#2401)
add 823b505ab5 HDDS-5326. EC: Create a new as many racks as possible placement policy for EC (#2497)
add 90c4d9286d HDDS-5470 : EC: Add padding and generate parity if the last stripe is not full (#2455)
add 2df6c9caf5 Merge branch 'master' into ec branch
add e08d5009ab Removed SCMPipelineManager.java which should have been removed via HDDS-5554, but was somehow missing on an earlier master to ec branch merge.
add 6b17c7570d HDDS-5745. EC: Pipeline builder should copy replica indexes from original pipeline (#2642)
add 1d3423de76 HDDS-5477. EC: commit key should consolidate and create one keyLocationInfo per blockGrp (#2648)
add ebdd1f5e3d HDDS-5682: EC: Provide replication config option from CLI when creating bucket. (#2639)
add b55107fe76 HDDS-5550. EC: Adapt KeyInputStream to read EC Blocks (#2653)
add 0a0e62e002 EC: Add Codec and Stripe Size to ECReplicationConfig (#2674)
add fc17518932 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
add 5cf251654e HDDS-5741. EC: Remove hard coded chunksize and get from from ReplicationConfig (#2687)
add 831b3faa27 HDDS-5833: EC: Fix TestRootedOzoneFileSystem.testBucketDefaultsShouldBeInheritedToFileForEC failure in branch (#2718)
add 7a67f63429 HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
add a16ee9d2fb Merge branch 'master' into HDDS-3816-ec
add aee577c171 HDDS-5825: EC: ECKeyOutputStream#close fails if we write the partial chunk (#2720)
add f478b96360 HDDS-5832. EC: ECKeyOutputStream persists blocks in random order (#2717)
add d37fe6901b HDDS-5552. EC: Implement seek on ECBlockInputStream (#2723)
add 38e4cca273 HDDS-5755 EC: Refactor ECBlockOutputStreamEntry to accommodate all block group related ECBlockOuputStreams. (#2702)
add 914ce67af3 HDDS-5364: EC: Adopt EC related utility from Hadoop source repository (#2733)
add f5dbe00309 Hdds 5491: EC: Write should handle node failures. (#2767)
add c9fa9bdf0f HDDS-5874 EC: Integrate the Codec changes into EC Streams. (#2777)
add 818118a7ad HDDS-5948: EC: Fix the compile issue in TestOzoneECClient (Due to concurrent commits) (#2812)
add e82682fe8b HDDS-5551. EC: Implement an Input Stream to reconstruct EC blocks on demand (#2797)
add f85ac376e6 HDDS-5950. EC: Implement seek on ECBlockReconstructedStripeInputStream (#2818)
add 438799ba75 HDDS-5953: EC: Review the current flush API and clean up (#2828)
add 52f05fc349 HDDS-5951. EC: ECBlockReconstructedStripeInputStream should handle block read failures and continue reading (#2831)
add 67043109c0 Merging master into EC branch
add d5ac6738b7 HDDS-6012: EC: Fix TestOzoneShellHA failures post master merge with EC branch
add 9a7025a39b HDDS-6035: EC: Fix TestOmMetrics in merge branch
add a0349bd9a9 HDDS-6029: EC: Fix the replication config handling in OMDirectoryCreateRequest#dirKeyInfoBuilderNoACL (#1)
add 685442c6b8 HDDS-6001. EC: Create ECBlockReconstructedInputStream to wrap ECBlockReconstructedStripeInputStream (#2848)
add 93fadc9789 HDDS-5979: EC: Change CLI bucket default replication option name to -type (#2873)
add 6e7d0dc206 HDDS-6009. EC: Optimize ECBlockReconstructedStripeInputStream where there are no missing data indexes (#2872)
add 1b9724d60a HDDS-5955: EC: Track the failed servers to add into the excludeList when invoking allocateBlock (#2849)
add 460aa9b89b HDDS-6020: EC: Provide set replicationConfig option to bucket (#2880)
add a1d7292bcf HDDS-6010. EC: Create ECBlockInputStreamProxy to choose between reconstruction and normal reads (#2889)
add 73bddf36d0 Updates to address compile issues after merge
add 3007b5c68d HDDS-6081. EC: Fix ReplicationConfig related test failures that happened due to merging HDDS-5997 to the EC branch
add c6604833d1 HDDS-5952. EC: ECBlockReconstructedStripeInputStream should read from blocks in parallel (#2899)
add 86f735418a HDDS-6036: EC: handleStripeFailure should retry (#2910)
add f8fd2ff66b HDDS-6065: EC: Provide CLI option to reset the bucket replication config (#2927)
add ba192b1de1 HDDS-6132: EC: HandleStripeFailure should not release the cachebuffers. (#2940)
add 3eb55db014 HDDS-5949. EC: Create reusable buffer pool shared by all EC input and output streams (#2929)
add 5856411168 HDDS-6179. EC: Provide toString in ECReplicationConfig (#2986)
add 07b2338525 HDDS-6180. EC: Pipeline creator should ignore creating pipelines for ZERO factor (#2985)
add aaf6f17045 HDDS-6021: EC: Client side exclude nodes list should expire after certain time period or based on the list size. (#2973)
add d0467bac1d HDDS-6220. EC: Introduce a gRPC client implementation for EC with really async WriteChunk and PutBlock. (#3016)
add b40ab3f97b HDDS-6172: EC: Document the Ozone EC (#3006)
add 78ccdd540c HDDS-6131: EC: Replication config from bucket should be refreshed in o3fs (#3008)
add 6b5c01c843 HDDS-6184. EC: put command should create EC key if bucket is EC (#2990)
add 4d660c4d79 HDDS-6233. EC: Bucket does not display correct EC replication details (#3025)
add 2e60a4f8cb Merge master to EC Branch
add 58f8e30538 HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist (#3029)
add 78ca097eac HDDS-6083: EC: Fix flakyness of tests around nodefailures (#3021)
add dfa8141937 HDDS-6237: EC: putBlock should pass close flag true on end of block group/close file. (#3026)
add ee630e2995 HDDS-6240. EC: Container Info command with json switch fails for EC containers (#3033)
add 8e52561eab HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3037)
add ddcfc521af HDDS-6252. EC: Datanode Chunk Validator fails on encountering EC pipeline (#3035)
add ca6e4c3599 HDDS-6255. EC: Replication Manager should skip EC Containers (#3038)
add 5d2fa94949 HDDS-6256. EC: Parity blocks are incorrectly padded with zeros to the chunk size (#3043)
add 9a0f812b3b HDDS-6231. EC: Create EC acceptance test environment and some basic tests (#3030)
add 722c444aea HDDS-6250. EC: Add replica index to the output in the container info command (#3041)
add 29c69b109b HDDS-6272: EC: Recon UI failing because of EC replication factor (#3067)
add 22e34820cf HDDS-6258. EC: Read with stopped but not dead nodes gives IllegalStateException rather than InsufficientNodesException (#3048)
add c4aec7bb97 HDDS-6259. EC: Pipelines for closed containers should contain correct replica indexes (#3062)
add dafdb23255 HDDS-6295. EC: Fix unaligned stripe write failure due to length overflow. (#3076)
add 2d63cc4ec9 HDDS-6194. EC: Freon ockg support EC write (#2996)
add 79f60c4d5c HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC libraries (#3087)
add aebaff55a2 Merge master to EC branch HDDS-3816-ec
add b2a33b759e HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
add 4b9d38763a HDDS-6294. EC: Make cluster-wide EC configuration take effect (#3089)
add 550eadc970 HDDS-6308. EC: Handle Replication Factor to consider EC Config in Recon UI (#3078)
add 6f64eb77d5 HDDS-6342. EC: Fix large write with multiple stripes upon stripe failure. (#3108)
add 60a41bc85e HDDS-6249. EC: Fix todo items in TestECKeyOutputStream (#3102)
add 1d23619164 HDDS-6188: EC: Adapt java side native coder classes from hadoop. (#3046)
add 387fb1a2c1 HDDS-6232. EC: Update help strings for replication config (#3113)
add 0a83e7018e HDDS-6341: EC: Fix the race condition in TestECBlockReconstructedStripeInputStream. (#3111)
add aac07ef790 HDDS-6347. EC: Freon randomKeys EC key support (#3112)
add 6b8d935f73 HDDS-6319. EC: Fix read big file failure with EC policy 10+4. (#3094)
add b9adbcd89a HDDS-6348: EC: PartialStripe failure handling logic is writing padding bytes also to DNs (#3124)
add b90f00192d HDDS-6372. EC: Do not throw NotImplementedException in flush() (#3133)
add 78ca327e77 HDDS-6358. EC: Refactor ECKeyOutputStream#write() (#3120)
add 9e3cf98a30 HDDS-6279. EC: Calculate EC replication correctly when updating bucket usage (#3136)
add e675e3ba3d HDDS-6380. EC: Key Info command should not display legacy replication fields as they duplicate ReplicationConfig (#3141)
add 9ee6d72f31 HDDS-6373. EC: Exclude pipeline upon container close instead of exclude DNs. (#3137)
add 38409ea5a0 HDDS-6364. EC: Discard pre-allocated blocks to eliminate worthless retries. (#3127)
add 9680edf850 HDDS-6384. EC: Ensure EC container usage is updated correctly when handling reports (#3147)
add 58eac935cd HDDS-6185: EC: EC keys can't be created via S3 interfaces (#3150)
add 8cff9fd2ad HDDS-6368. EC: Fix broken future chain and cleanup unnecessary validation function. (#3128)
add 8776430812 HDDS-6382. EC: ListPipelines command should consider EC Containers (#3161)
add 28482bf2b9 HDDS-6422. EC: Fix too many idle threads during reconstruct read. (#3168)
add 074203fc0a HDDS-6437. EC: Avoid allocating buffers in EC Reconstruction Streams until first read (#3181)
add 9675863705 HDDS-6413. EC: OzoneManagerRequestHandler needs to handle ECReplicationConfig (#3198)
add 06f1eb5ad6 HDDS-6248. EC: Container list command should allow filtering of EC containers (#3179)
add 03ff3f9592 HDDS-6445. EC: Fix allocateBlock failure due to inaccurate excludedNodes check. (#3190)
add af48c537a9 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
add 217c1911b2 EC: Check isFullCell inside handleDataWrite (#3202)
add 9cda6a1a06 HDDS-6460. EC: Adjust requested size by EC DataNum in WritableECContainerProvider (#3203)
add 04c62c53f2 HDDS-6411. EC: OmMultipartKeyInfo needs to handle ECReplicationConfig (#3210)
add fc6ac0c48c HDDS-6448. EC: Reconstructed Input Streams should free resources after reading to end of block (#3197)
add f386662a37 HDDS-6381. EC: Improve exception message in ByteBufferEncodingState (#3143)
add 9ec90f5820 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec-merge-master
add c55f30edf0 HDDS-6412: EC: Handle ECReplicationConfig in initiateMultipartUpload API (#3184)
add 16c3e99eae HDDS-6478. EC: Fix ISA-l load hadoop native lib UnsatisfiedLinkError (#3218)
add 143315c89b HDDS-6409. EC: OzoneMultipartUpload needs to handle ECReplicationConfig (#3279)
add d9529b32ea HDDS-6434: EC: Overwriting an EC key with a Ratis key fails (#3274)
add df96b86c8d HDDS-6557. EC: Execute S3 acceptance tests with EC (#3281)
add e9b6215210 Merge master to EC Branch HDDS-3816-ec
add e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
add 6c6b740a47 HDDS-6209. EC: [Forward compatibility issue] New client to older server could fail due to the unavailability for client default replication config (#3267)
add 70b547be23 HDDS-5909. EC: Onboard EC into upgrade framework (#3262)
add cb0a84b27c Merge master to EC branch HDDS-3816-ec
add 96669856ad Merge EC branch HDDS-3816-ec to master
add 681d0a8f5e HDDS-6471. Only repeat flaky tests (#3247)
add dc9da7d405 HDDS-6606. Rat check failure ignored for ozone-annotation-processing (#3321)
add be5888b907 HDDS-6580. Introduce KEY_PATH_LOCK under OzoneManagerLock.Resource (#3308)
add 4345fb1c58 HDDS-6189. Intermittent failure in TestDirectoryDeletingServiceWithFSO (#3328)
add 03b0a61958 HDDS-6583. Introduce lock strategy pattern implementations based on configuration flags (#3310)
add cb2754fd9c HDDS-6555. Container Deletion should not depend on usedBytes being zero (#3276)
add b077cc6123 HDDS-6561. Modify default bucket layout to LEGACY. (#3296)
add 9ac8fbce70 HDDS-6614. EC: Fix Datanode block file INCONSISTENCY during heavy load. (#3323)
add 2876c54b8d HDDS-6405. Selective checks: run rat for readme change (#3156)
add 497c7a40b8 HDDS-6617. Skip compile(11) for non-java changes (#3324)
add b8388b096a HDDS-6639. Remove flaky tag for testOMProxyProviderFailoverOnConnectionFailure (#3343)
add b78adc4c71 HDDS-6620. Gather AuditLog params for S3g (#3325)
add de9fcca1c3 HDDS-6522. Fix TestOzoneManagerHAWithData#testTwoOMNodesDown (#3341)
add 2ea9e78bf1 Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
add edec164302 HDDS-6643. Minor fixes for OM gRPC feature (#3347)
add db7f691855 HDDS-6651. Restore S3 Gateway config key check in TestOzoneConfigurationFields (#3354)
add db778fce96 HDDS-6457. Add freon command to run r/w mix workload using FS (#3266)
add c6314b987c HDDS-6512. Install Hugo during build if needed (#3287)
add f575e238f8 HDDS-6553. Incorrect timeout when checking balancer iteration result. (#3272)
add 9d771eb6ed HDDS-6647. Tag some known flaky tests (#3349)
add b4e8e7ac8d HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
add 73d4876dfe Revert "HDDS-6579. Add commandline argument for Container Info command. (#3306)" (#3336)
add f49a691b41 HDDS-6626. Exclude Hadoop Thirdparty from shaded Ozone FS jars (#3330)
add 4476a5be19 HDDS-6632. ReplicationManagerReport should handle unknown metric in protobuf (#3335)
add d9a623ad72 HDDS-6598. Add a BackgroundPipelineScrubber to scrub all pipelines. (#3337)
add 1944b2cda9 HDDS-6624. Make ECBlockReconstructedStripeInputStream available for datanodes (#3353)
add 240002559e Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
add 124723a4b2 HDDS-6661. TestBackgroundPipelineScrubber.testRun() fails intermittently (#3359)
add c251258832 HDDS-6658. BackgroundPipelineCreator does not always stop quickly (#3357)
add 81b6bbad14 HDDS-6641. datanode usageinfo CLI should provide JSON output option (#3356)
add 59f60089bb HDDS-6653. Small fixes in PipelineStateMap (#3355)
add d2ac33627a HDDS-6567. Store datanode command queue counts from heartbeat in DatanodeInfo in SCM (#3329)
add d53cb2c1b7 HDDS-6529. Adding Unit-Test cases for S3-Gateway Object-Endpoint Metrics (#3302)
add ba9f3425ba HDDS-6684. Move unit tests out of integration (#3375)
add deb88f35ec HDDS-6586: EC: Implement the EC Reconstruction Command with necessary information (#3345)
add e70bfd0689 Revert "HDDS-5740. Enable ratis by default for SCM. (#2637)" (#3362)
add 28170f2672 HDDS-6685. Follower OM crashed when validating S3 auth info. (#3376)
add c33f6a58d8 HDDS-6640. Node.isAncestor might return incorrect result (#3342)
add c6cd84cc70 HDDS-6377. Redundant loop while doing triggerHeartbeat in DatanodeStateMachine (#3139)
add 71ce98081d HDDS-6673. Reduce OzoneFS test combinations in ozonesecure (#3368)
add c99fd20da9 HDDS-6589. Add a new replication manager and change the existing one to legacy (#3352)
add a3f5021b23 HDDS-6669. Exercise OM in ozonescripts environment (#3367)
add 9ff16ac551 HDDS-6655. Use injected clocks in PipelineManagerImpl and BackgroundPipelineScrubber to ease testing (#3363)
add f315500c73 HDDS-6551. Introduce StatefulService in scm (#3307)
add 9055a11216 HDDS-6556. Check quota while allocate and update usedBytes when commit. (#3286)
add a9df285447 HDDS-6563. Revert "HDDS-5867. Update quota when deleting open keys" (#3282)
add 3889373070 HDDS-6703. Disable failing testPrepareDownedOM until Ratis upgrade (#3387)
add d9c56aec00 HDDS-6675. Avoid repeating some S3 tests for different bucket types (#3370)
add 73e69035e0 HDDS-6659: EC: Add BlockGroupLen info as part of PutBlock in EC Writes for helping in recovery. (#3380)
add e90a44de2f HDDS-6602. Incorrect logic in PipelineProvider.pickNodesNotUsed (#3318)
add d2c5631d2f HDDS-6716. Adopt RATIS-1560 (#3394)
add 6e0c85eb43 HDDS-6715. Avoid using GrpcClientProtocolService.LOG (#3393)
add d4d8e3c4e9 HDDS-6666. Make Hugo markdown image syntax add responsive image class (#3364)
add ff351025bc HDDS-6709. Fix bucket usedBytes while versioning is true. (#3388)
add 1419799bad HDDS-6678. Use secure random in production code (#3374)
add bfc361e1b8 HDDS-6507. Spotbugs transitive dependencies may result in NoClassDefFound error (#3236)
add a32549cbc1 HDDS-6708. [FSO] Make normalize keyPath on key rename request (#3386)
add 88b5e854be HDDS-6400. EC: Refactor ECKeyOutputStream for better code reuse (#3372)
add bccb8c2c83 HDDS-6679. Further splits integration tests (#3373)
add 5459b49538 HDDS-6720. Add a function to get bucket layout info in Request Validators (#3395)
add 91757d5145 HDDS-6734. ozone admin pipeline list CLI is not backward compatible (#3406)
add 9f0766ecf1 HDDS-6619. Add freon command to run r/w mix workload using ObjectStore APIs (#3383)
add 6edcc13900 HDDS-6680. Pre-Finalize behaviour for Bucket Layout Feature. (#3377)
add 86ffd94735 HDDS-6704. EC: ReplicationManager - create version of ContainerReplicaCounts applicable to EC (#3405)
add 3b66583c6f HDDS-6560. Add general Grafana dashboard (#3285)
add a6afe188fd HDDS-6630. Explicitly remove node after being chosen (#3332)
add 4046150447 HDDS-6730. Migrate tests in hdds-tools to JUnit5 (#3402)
add f131de5f99 HDDS-5919. In kubernetes OM HA has circular dependency on service availability (#3185)
add 340f5661e4 HDDS-6731. Migrate simple tests in hdds-server-framework to JUnit5 (#3412)
add 690b3cd338 HDDS-6748. Intermittent timeout in TestECBlockReconstructedInputStream#testReadDataWithUnbuffer (#3416)
add ed470082e6 HDDS-6742. Audit operation category mismatch (#3407)
add 71b4feb0ba HDDS-6657. Improve Ozone integrated Ranger configuration instructions (#3365)
add fb0917328b HDDS-6749. SCM includes itself as peer in addSCM request (#3413)
add 5ba03daab7 HDDS-6750. Migrate simple tests in hdds-server-scm to JUnit5 (#3417)
add 44cf924f4d HDDS-6660: EC: Add the DN side Reconstruction Handler class. (#3399)
add e169d71467 HDDS-6737. Migrate parameterized tests in hdds-server-framework to JUnit5 (#3414)
add a26ad30f6b HDDS-6596. EC: Support ListBlock from CoordinatorDN (#3410)
add f328aa033b HDDS-6491. Support FSO keys in getExpiredOpenKeys (#3226)
add 1067d353c0 HDDS-6650. S3MultipartUpload support update bucket usedNamespace. (#3404)
add d5e2b758f0 HDDS-6738. Migrate tests with rules in hdds-server-framework to JUnit5 (#3415)
add 78a018814c HDDS-6744. EC: ReplicationManager - create ContainerReplicaPendingOps class and integrate with ContainerManager (#3425)
add d6ece18f28 HDDS-6097. Upgrade to Ratis 2.3.0, gRPC 1.44.0, Netty 4.1.74 (#3433)
add 6cf05906ef HDDS-6777. Upgrade JUnit 5 to 5.8.2 (#3437)
add a6994b2d87 HDDS-6787. Bump gson to 2.8.9 (#3441)
add 2b8c4b7324 HDDS-6779. Register metricSink after initialization in S3 Gateway (#3436)
add fee0de9dd4 HDDS-6784. Missing close() for two RocksDB table iterators (#3440)
add 51ee649a15 HDDS-6778. Bump Guava to 31.1 (#3435)
add be1180f4b1 HDDS-6786. Bad value for metric NumBytesCommittedCount in testContainerStateMachineMetrics (#3443)
add ddb1fe24ab HDDS-6756. NPE raised by RpcClient if ServiceInfo certificates empty (#3428)
add c01fc7907a HDDS-6674. Create compat acceptance split (#3369)
new 79a812af1f Merge remote-tracking branch 'asf/master' into HDDS-4944
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 56 +-
dev-support/annotations/pom.xml | 4 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 201 ++
.../hdds/scm/storage/ExtendedInputStream.java | 91 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 +
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 +
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 ++
.../io/ECBlockReconstructedStripeInputStream.java | 632 +++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io/package-info.java | 0
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 +
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++
.../client/io/TestECBlockInputStreamProxy.java | 396 +++
.../io/TestECBlockReconstructedInputStream.java | 377 +++
.../TestECBlockReconstructedStripeInputStream.java | 721 ++++++
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 20 +
.../hdds/client/DefaultReplicationConfig.java | 134 +
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 20 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../hadoop/hdds/freon/FakeClusterTopology.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 66 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 -
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 +
.../common/src/main/resources/ozone-default.xml | 111 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 +
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +-
.../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 +
.../hdds/scm/container/TestContainerInfo.java | 86 +
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 +
.../container/common/helpers/TestExcludeList.java | 67 +
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +-
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +-
.../common/src/test/resources/auditlog.properties | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/report/ContainerReportPublisher.java | 1 +
.../common/report/PipelineReportPublisher.java | 1 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 78 +
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../ECReconstructionCommandInfo.java | 82 +
.../ECReconstructionCoordinatorTask.java | 52 +
.../reconstruction/ECReconstructionSupervisor.java | 72 +
.../container/ec/reconstruction/package-info.java | 34 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 1 +
.../commands/ReconstructECContainersCommand.java | 193 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../TestECReconstructionSupervisor.java | 53 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../docs/content/security/SecurityWithRanger.md | 7 +
.../docs/content/security/SecurityWithRanger.zh.md | 9 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 -
hadoop-hdds/docs/pom.xml | 12 +-
.../_markup/render-image.html} | 3 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 +
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 +
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 +
.../rawcoder/AbstractNativeRawEncoder.java | 99 +
.../rawcoder/ByteArrayDecodingState.java | 132 +
.../rawcoder/ByteArrayEncodingState.java | 105 +
.../rawcoder/ByteBufferDecodingState.java | 143 ++
.../rawcoder/ByteBufferEncodingState.java | 110 +
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 +
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 +
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 +
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 +
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 +
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 +
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 +
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 +
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 +
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 +
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 +
.../ozone/erasurecode/TestCodecRegistry.java | 159 ++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 +
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 +
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++
.../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 +
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 4 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../SCMBlockLocationFailoverProxyProvider.java | 3 +
.../client/DefaultCertificateClient.java | 9 +-
.../exceptions/TestSCMExceptionResultCodes.java | 19 +-
.../token/TestOzoneBlockTokenIdentifier.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 4 +-
.../certificate/authority/TestDefaultCAServer.java | 40 +-
.../certificate/authority/TestDefaultProfile.java | 22 +-
.../client/TestCertificateClientInit.java | 73 +-
.../client/TestDefaultCertificateClient.java | 60 +-
.../x509/certificate/utils/TestCRLCodec.java | 26 +-
.../certificate/utils/TestCertificateCodec.java | 41 +-
.../certificates/TestCertificateSignRequest.java | 60 +-
.../x509/certificates/TestRootCertificate.java | 68 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 14 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 49 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 28 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 28 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hdds/server/events/TestEventWatcher.java | 46 +-
.../hdds/server/http/TestBaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +-
.../hdds/server/http/TestHttpRequestLog.java | 16 +-
.../server/http/TestHttpRequestLogAppender.java | 12 +-
.../hadoop/hdds/server/http/TestHttpServer2.java | 4 +-
.../hdds/server/http/TestProfileServlet.java | 39 +-
.../server/http/TestPrometheusMetricsSink.java | 47 +-
.../server/http/TestRatisDropwizardExports.java | 13 +-
.../hdds/server/http/TestRatisNameRewrite.java | 65 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 78 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +-
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +-
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 38 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/ECContainerReplicaCount.java | 313 +++
.../scm/container/balancer/ContainerBalancer.java | 135 +-
.../balancer/ContainerBalancerMetrics.java | 70 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 +++
.../container/replication/ContainerReplicaOp.java | 63 +
.../replication/ContainerReplicaPendingOps.java | 227 ++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 +--
.../replication/ReplicationActivityStatus.java | 104 -
.../container/replication/ReplicationManager.java | 453 ++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 +
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 +
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 +
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 31 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 ++
.../algorithms/RandomPipelineChoosePolicy.java | 1 +
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../scm/server/StorageContainerManagerStarter.java | 3 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 12 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +-
.../command/TestCommandStatusReportHandler.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 39 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 109 +-
.../scm/container/TestContainerReportHandler.java | 207 +-
.../scm/container/TestContainerStateManager.java | 18 +-
.../TestIncrementalContainerReportHandler.java | 34 +-
.../hdds/scm/container/TestReplicationManager.java | 573 +++--
.../scm/container/TestUnknownContainerReport.java | 10 +-
.../container/balancer/TestContainerBalancer.java | 55 +-
.../container/balancer/TestFindTargetStrategy.java | 26 +-
.../algorithms/TestContainerPlacementFactory.java | 43 +-
.../TestContainerPlacementStatusDefault.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 28 +-
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +--
.../TestSCMContainerPlacementRandom.java | 36 +-
.../TestContainerReplicaPendingOps.java | 226 ++
.../replication/TestReplicationManagerMetrics.java | 27 +-
.../states/TestContainerReplicaCount.java | 12 +-
.../states/TestECContainerReplicaCount.java | 432 ++++
.../hdds/scm/ha/TestReplicationAnnotation.java | 20 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +-
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 89 +
.../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +-
...ler.java => TestCommandQueueReportHandler.java} | 106 +-
.../hdds/scm/node/TestContainerPlacement.java | 7 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 10 +-
.../hdds/scm/node/TestNodeReportHandler.java | 22 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 12 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 +
.../scm/pipeline/TestRatisPipelineProvider.java | 93 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 44 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++
.../TestPipelineChoosePolicyFactory.java | 14 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 26 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 23 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../scm/server/TestSCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +-
.../server/TestStorageContainerManagerStarter.java | 18 +-
.../ozone/container/common/TestEndPoint.java | 84 +-
.../placement/TestContainerPlacement.java | 10 +-
.../container/placement/TestDatanodeMetrics.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 21 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +-
.../org/apache/ozone/test/GenericTestUtils.java | 1 +
hadoop-hdds/tools/pom.xml | 7 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +-
.../scm/cli/container/TestReportSubCommand.java | 36 +-
.../datanode/TestContainerBalancerSubCommand.java | 14 +-
.../cli/datanode/TestDecommissionSubCommand.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 14 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 16 +-
.../cli/datanode/TestRecommissionSubCommand.java | 14 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 +
.../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 +
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 +-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 ++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +-
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++
.../ozone/om/ha/OMFailoverProxyProvider.java | 22 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++
.../src/test/resources/log4j.properties | 0
hadoop-ozone/csi/pom.xml | 17 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 11 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 12 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 +
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../buckethead.robot => basic/single_node.robot} | 22 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 +
.../dist/src/main/smoketest/ec/basic.robot | 64 +
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 1 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 1 +
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/s3/webui.robot | 1 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 2 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
hadoop-ozone/insight/pom.xml | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 9 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++--
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +-
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 +
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 44 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
hadoop-ozone/ozone-manager/pom.xml | 13 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 10 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 316 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 +
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 +
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 57 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 67 +-
.../request/file/OMFileCreateRequestWithFSO.java | 19 +-
.../om/request/key/OMAllocateBlockRequest.java | 52 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 72 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 62 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 26 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCompleteRequest.java | 60 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/request/validation/ValidationContext.java | 24 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 14 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 5 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 13 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 107 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 -
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 99 +
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 55 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadCompleteRequest.java | 8 +
.../request/validation/TestRequestValidations.java | 95 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 59 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/pom.xml | 12 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 8 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++
.../hadoop/ozone/s3/OzoneClientProducer.java | 41 +-
.../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 +
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++
hadoop-ozone/tools/pom.xml | 2 +-
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 +
.../hadoop/ozone/freon/ContentGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../ozone/freon/FreonReplicationOptions.java | 73 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 +
.../ozone/freon/HadoopNestedDirGenerator.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 108 +
.../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 +
.../containergenerator/GeneratorDatanode.java | 1 +
.../hadoop/ozone/shell/ReplicationOptions.java | 87 +
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 87 +-
685 files changed, 40027 insertions(+), 5756 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ECXceiverClientGrpc.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java => client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockExtendedInputStream.java} (68%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ExtendedInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java} (58%)
copy {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java (100%)
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/ECStreamTestUtil.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStreamProxy.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockLocationInfo.java (54%)
rename hadoop-hdds/{framework => common}/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java (100%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/OzoneNetUtils.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestECReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestSCMHAUtils.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerInfo.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/common/helpers/TestExcludeList.java
copy hadoop-hdds/{server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java => common/src/test/java/org/apache/hadoop/ozone/ha/TestOzoneNetUtils.java} (54%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCommandInfo.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinatorTask.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionSupervisor.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/RequestIdentifier.java => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/package-info.java (57%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/protocol/commands/TestReconstructionECContainersCommands.java
create mode 100644 hadoop-hdds/docs/content/feature/EC-Chunk-Layout.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reads-With-No-Failures.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reconstructional-Read.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Write-Block-Allocation-in-Containers.png
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.md
delete mode 100644 hadoop-hdds/docs/dev-support/bin/make_images_responsive.py
rename hadoop-hdds/docs/themes/ozonedoc/layouts/{shortcodes/image.html => _default/_markup/render-image.html} (84%)
copy {hadoop-ozone/client => hadoop-hdds/erasurecode}/pom.xml (57%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/HadoopNativeECAccessorUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder}/package-info.java (73%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/BufferAllocator.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/CodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/ECChunk.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode}/package-info.java (69%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/CoderUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/EncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ErasureCodeNative.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/package-info.java} (50%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/CodecUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GF256.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GaloisField.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/RSUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util}/package-info.java (74%)
create mode 100644 hadoop-hdds/erasurecode/src/main/resources/META-INF/services/org.apache.ozone.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCoderBase.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/package-info.java} (77%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderBenchmark.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestCodecRawCoderMapping.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestDummyRawCoder.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeRSRawCoder.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java} (57%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoder.java} (69%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoderBase.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawCoderBase.java
copy hadoop-hdds/{framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java} (53%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoder.java} (75%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder}/package-info.java (83%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{ReplicationActivityStatusMXBean.java => InflightAction.java} (58%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ReplicationManager.java => replication/LegacyReplicationManager.java} (88%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatus.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ByteStringCodec.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
copy hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/{TestSCMContainerPlacementRackAware.java => TestSCMContainerPlacementRackScatter.java} (70%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestStatefulServiceStateManagerImpl.java
copy hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/{TestNodeReportHandler.java => TestCommandQueueReportHandler.java} (53%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateMap.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/pipeline/TestListPipelinesSubCommand.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntryPool.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/BlockStreamAccessor.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestKeyInputStreamEC.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
copy hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java => common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransportFactory.java} (58%)
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestQuotaUtil.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
copy hadoop-ozone/{integration-test => common}/src/test/resources/log4j.properties (100%)
create mode 100644 hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Overall Metrics.json
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
create mode 100644 hadoop-ozone/dist/src/main/org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory
copy hadoop-ozone/dist/src/main/smoketest/{s3/buckethead.robot => basic/single_node.robot} (58%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OBSKeyPathLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/RegularBucketLockStrategy.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
copy hadoop-ozone/{dev-support/checks/build.sh => s3gateway/src/main/resources/META-INF/services/org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory} (76%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/AbstractOmBucketReadWriteOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteFileOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
copy hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java => tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java} (62%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into HDDS-4944
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 79a812af1f1156aed729e82a85b9492c00ef573f
Merge: 35043683a6 c01fc7907a
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Mon May 23 12:30:16 2022 -0700
Merge remote-tracking branch 'asf/master' into HDDS-4944
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
Change-Id: I05134acea37376d41d37bced6e40c700fbd9a0bf
.github/workflows/post-commit.yml | 56 +-
dev-support/annotations/pom.xml | 4 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 201 ++
.../hdds/scm/storage/ExtendedInputStream.java | 91 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 +
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 +
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 ++
.../io/ECBlockReconstructedStripeInputStream.java | 632 +++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io}/package-info.java | 6 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 +
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++
.../client/io/TestECBlockInputStreamProxy.java | 396 +++
.../io/TestECBlockReconstructedInputStream.java | 377 +++
.../TestECBlockReconstructedStripeInputStream.java | 721 ++++++
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 20 +
.../hdds/client/DefaultReplicationConfig.java | 134 +
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 20 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../hadoop/hdds/freon/FakeClusterTopology.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 66 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 -
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 +
.../common/src/main/resources/ozone-default.xml | 111 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 +
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +-
.../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 +
.../hdds/scm/container/TestContainerInfo.java | 86 +
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 +
.../container/common/helpers/TestExcludeList.java | 67 +
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +-
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +-
.../common/src/test/resources/auditlog.properties | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/report/ContainerReportPublisher.java | 1 +
.../common/report/PipelineReportPublisher.java | 1 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 78 +
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../ECReconstructionCommandInfo.java | 82 +
.../ECReconstructionCoordinatorTask.java | 52 +
.../reconstruction/ECReconstructionSupervisor.java | 72 +
.../container/ec/reconstruction/package-info.java | 34 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 1 +
.../commands/ReconstructECContainersCommand.java | 193 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../TestECReconstructionSupervisor.java | 53 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../docs/content/security/SecurityWithRanger.md | 7 +
.../docs/content/security/SecurityWithRanger.zh.md | 9 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 -
hadoop-hdds/docs/pom.xml | 12 +-
.../_markup/render-image.html} | 3 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 +
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 +
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 +
.../rawcoder/AbstractNativeRawEncoder.java | 99 +
.../rawcoder/ByteArrayDecodingState.java | 132 +
.../rawcoder/ByteArrayEncodingState.java | 105 +
.../rawcoder/ByteBufferDecodingState.java | 143 ++
.../rawcoder/ByteBufferEncodingState.java | 110 +
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 +
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 +
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 +
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 +
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 +
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 +
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 +
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 +
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 +
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 +
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 +
.../ozone/erasurecode/TestCodecRegistry.java | 159 ++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 +
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 +
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++
.../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 +
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 4 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../SCMBlockLocationFailoverProxyProvider.java | 3 +
.../client/DefaultCertificateClient.java | 9 +-
.../exceptions/TestSCMExceptionResultCodes.java | 19 +-
.../token/TestOzoneBlockTokenIdentifier.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 4 +-
.../certificate/authority/TestDefaultCAServer.java | 40 +-
.../certificate/authority/TestDefaultProfile.java | 22 +-
.../client/TestCertificateClientInit.java | 73 +-
.../client/TestDefaultCertificateClient.java | 60 +-
.../x509/certificate/utils/TestCRLCodec.java | 26 +-
.../certificate/utils/TestCertificateCodec.java | 41 +-
.../certificates/TestCertificateSignRequest.java | 60 +-
.../x509/certificates/TestRootCertificate.java | 68 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 14 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 49 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 28 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 28 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hdds/server/events/TestEventWatcher.java | 46 +-
.../hdds/server/http/TestBaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +-
.../hdds/server/http/TestHttpRequestLog.java | 16 +-
.../server/http/TestHttpRequestLogAppender.java | 12 +-
.../hadoop/hdds/server/http/TestHttpServer2.java | 4 +-
.../hdds/server/http/TestProfileServlet.java | 39 +-
.../server/http/TestPrometheusMetricsSink.java | 47 +-
.../server/http/TestRatisDropwizardExports.java | 13 +-
.../hdds/server/http/TestRatisNameRewrite.java | 65 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 78 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +-
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +-
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 38 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/ECContainerReplicaCount.java | 313 +++
.../scm/container/balancer/ContainerBalancer.java | 135 +-
.../balancer/ContainerBalancerMetrics.java | 70 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 +++
.../container/replication/ContainerReplicaOp.java | 63 +
.../replication/ContainerReplicaPendingOps.java | 227 ++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 +--
.../replication/ReplicationActivityStatus.java | 104 -
.../container/replication/ReplicationManager.java | 453 ++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 +
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 +
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 +
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 31 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 ++
.../algorithms/RandomPipelineChoosePolicy.java | 1 +
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../scm/server/StorageContainerManagerStarter.java | 3 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 12 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +-
.../command/TestCommandStatusReportHandler.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 39 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 109 +-
.../scm/container/TestContainerReportHandler.java | 207 +-
.../scm/container/TestContainerStateManager.java | 18 +-
.../TestIncrementalContainerReportHandler.java | 34 +-
.../hdds/scm/container/TestReplicationManager.java | 573 +++--
.../scm/container/TestUnknownContainerReport.java | 10 +-
.../container/balancer/TestContainerBalancer.java | 55 +-
.../container/balancer/TestFindTargetStrategy.java | 26 +-
.../algorithms/TestContainerPlacementFactory.java | 43 +-
.../TestContainerPlacementStatusDefault.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 28 +-
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +--
.../TestSCMContainerPlacementRandom.java | 36 +-
.../TestContainerReplicaPendingOps.java | 226 ++
.../replication/TestReplicationManagerMetrics.java | 27 +-
.../states/TestContainerReplicaCount.java | 12 +-
.../states/TestECContainerReplicaCount.java | 432 ++++
.../hdds/scm/ha/TestReplicationAnnotation.java | 20 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +-
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 89 +
.../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +-
...ler.java => TestCommandQueueReportHandler.java} | 106 +-
.../hdds/scm/node/TestContainerPlacement.java | 7 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 10 +-
.../hdds/scm/node/TestNodeReportHandler.java | 22 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 12 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 +
.../scm/pipeline/TestRatisPipelineProvider.java | 93 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 44 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++
.../TestPipelineChoosePolicyFactory.java | 14 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 26 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 23 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../scm/server/TestSCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +-
.../server/TestStorageContainerManagerStarter.java | 18 +-
.../ozone/container/common/TestEndPoint.java | 84 +-
.../placement/TestContainerPlacement.java | 10 +-
.../container/placement/TestDatanodeMetrics.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 21 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +-
.../org/apache/ozone/test/GenericTestUtils.java | 1 +
hadoop-hdds/tools/pom.xml | 7 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +-
.../scm/cli/container/TestReportSubCommand.java | 36 +-
.../datanode/TestContainerBalancerSubCommand.java | 14 +-
.../cli/datanode/TestDecommissionSubCommand.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 14 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 16 +-
.../cli/datanode/TestRecommissionSubCommand.java | 14 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 +
.../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 +
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 +-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 ++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +-
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++
.../ozone/om/ha/OMFailoverProxyProvider.java | 22 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++
.../common/src/test/resources/log4j.properties | 21 +
hadoop-ozone/csi/pom.xml | 17 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 11 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 12 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 +
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../buckethead.robot => basic/single_node.robot} | 22 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 +
.../dist/src/main/smoketest/ec/basic.robot | 64 +
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 1 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 1 +
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/s3/webui.robot | 1 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 2 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
hadoop-ozone/insight/pom.xml | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 9 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++--
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +-
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 +
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 44 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
hadoop-ozone/ozone-manager/pom.xml | 13 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 10 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 316 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 +
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 +
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 57 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 67 +-
.../request/file/OMFileCreateRequestWithFSO.java | 19 +-
.../om/request/key/OMAllocateBlockRequest.java | 52 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 72 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 62 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 26 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCompleteRequest.java | 60 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/request/validation/ValidationContext.java | 24 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 14 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 5 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 13 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 107 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 -
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 99 +
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 55 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadCompleteRequest.java | 8 +
.../request/validation/TestRequestValidations.java | 95 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 59 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/pom.xml | 12 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 8 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++
.../hadoop/ozone/s3/OzoneClientProducer.java | 41 +-
.../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 +
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++
hadoop-ozone/tools/pom.xml | 2 +-
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 +
.../hadoop/ozone/freon/ContentGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../ozone/freon/FreonReplicationOptions.java | 73 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 +
.../ozone/freon/HadoopNestedDirGenerator.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 108 +
.../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 +
.../containergenerator/GeneratorDatanode.java | 1 +
.../hadoop/ozone/shell/ReplicationOptions.java | 87 +
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 87 +-
685 files changed, 40052 insertions(+), 5758 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f0a26df46f,f1c3d6578b..067777bb79
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -473,55 -455,6 +474,58 @@@ public final class OzoneConsts
public static final String DELEGATION_TOKEN_SERVICE = "service";
public static final String DELEGATION_TOKEN_RENEWER = "renewer";
+ // EC Constants
+ public static final String BLOCK_GROUP_LEN_KEY_IN_PUT_BLOCK = "blockGroupLen";
++
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users";
+
+ // Ideally we should use /addUsersAndGroups endpoint for add user to role,
+ // but it always return 405 somehow.
+ // https://ranger.apache.org/apidocs/resource_RoleREST.html
+ // #resource_RoleREST_addUsersAndGroups_PUT
+ public static final String OZONE_OM_RANGER_ADMIN_ROLE_ADD_USER_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_USER_HTTP_ENDPOINT =
+ "/service/xusers/users/?name=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_USER_HTTP_ENDPOINT =
+ "/service/xusers/secure/users/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/name/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_GROUP_HTTP_ENDPOINT =
+ "/service/xusers/secure/groups/id/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_ROLE_HTTP_ENDPOINT =
+ "/service/roles/roles/";
+
+ public static final String OZONE_OM_RANGER_ADMIN_CREATE_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyName=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_ID_HTTP_ENDPOINT =
+ "/service/public/v2/api/policy/?policyId=";
+
+ public static final String OZONE_OM_RANGER_ADMIN_DELETE_POLICY_HTTP_ENDPOINT =
+ "/service/plugins/policies/";
+
+ public static final String OZONE_OM_RANGER_OZONE_SERVICE_ENDPOINT =
+ "/service/plugins/services/";
+
+ public static final String OZONE_OM_RANGER_DOWNLOAD_ENDPOINT =
+ "/service/plugins/secure/policies/download/cm_ozone" +
+ "?supportsPolicyDeltas=true&lastKnownVersion=";
+
+ public static final String OZONE_OM_RANGER_ALL_POLICIES_ENDPOINT =
+ "/service/plugins/policies/service/";
+
+ public static final String OZONE_TENANT_RANGER_POLICY_LABEL = "OzoneTenant";
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3077bc73c1,c7764e9d5b..db0882c80b
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -77,8 -86,10 +86,11 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.TenantArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
+ import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
+ import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 291eb6f28d,bc1ed8d6f5..aa34f049f4
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@@ -312,44 -341,11 +341,52 @@@ public final class OMConfigKeys
public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT
= 10000;
+ /**
+ * This configuration shall be enabled to utilize the functionality of the
+ * fine-grained KEY_PATH_LOCK.
+ */
+ public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED =
+ "ozone.om.key.path.lock.enabled";
+ public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false;
++
+ public static final String OZONE_OM_MULTITENANCY_ENABLED =
+ "ozone.om.multitenancy.enabled";
+ public static final boolean OZONE_OM_MULTITENANCY_ENABLED_DEFAULT = false;
+
+ /**
+ * Temporary configuration properties for Ranger REST use in multitenancy.
+ */
+ public static final String OZONE_RANGER_OM_IGNORE_SERVER_CERT =
+ "ozone.om.ranger.ignore.cert";
+ public static final boolean OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT =
+ true;
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT =
+ "ozone.om.ranger.connection.timeout";
+ public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT =
+ "ozone.om.ranger.connection.request.timeout";
+ public static final String
+ OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_USER =
+ "ozone.om.ranger.https.admin.api.user";
+ // TODO: Note this should be removed once Ranger Java Client is in place.
+ // And Ranger SPNEGO auth (ranger.spnego.kerberos.principal ?) should be used
+ // instead. Or keep this solely for dev testing. See HDDS-5836.
+ public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD =
+ "ozone.om.ranger.https.admin.api.passwd";
+ public static final String OZONE_RANGER_HTTPS_ADDRESS_KEY =
+ "ozone.om.ranger.https-address";
+ public static final String OZONE_RANGER_SERVICE =
+ "ozone.om.ranger.service";
+
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL
+ = "ozone.om.multitenancy.ranger.sync.interval";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL_DEFAULT
+ = TimeDuration.valueOf(600, TimeUnit.SECONDS);
+ public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT
+ = "ozone.om.multitenancy.ranger.sync.timeout";
+ public static final TimeDuration
+ OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT_DEFAULT
+ = TimeDuration.valueOf(10, TimeUnit.SECONDS);
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 717df88b48,168b2c8bb4..a669f8a7df
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@@ -243,19 -243,6 +243,20 @@@ public class OMException extends IOExce
LAYOUT_FEATURE_FINALIZATION_FAILED,
PREPARE_FAILED,
NOT_SUPPORTED_OPERATION_WHEN_PREPARED,
- NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION
++ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+
+ TENANT_NOT_FOUND,
+ TENANT_ALREADY_EXISTS, INVALID_TENANT_ID,
+
+ ACCESS_ID_NOT_FOUND,
+ TENANT_USER_ACCESS_ID_ALREADY_EXISTS,
+ INVALID_TENANT_USERNAME,
+ INVALID_ACCESS_ID,
+ TENANT_AUTHORIZER_ERROR,
+
+ VOLUME_IS_REFERENCED,
+ TENANT_NOT_EMPTY,
+
+ FEATURE_NOT_ENABLED
}
}
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
index 0000000000,d54c9bc89b..f46f067914
mode 000000,100755..100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
@@@ -1,0 -1,84 +1,84 @@@
+ #!/usr/bin/env bash
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ source "$TEST_DIR"/testlib.sh
+
+ # Helper function, not a callback.
+ _check_hdds_mlvs() {
+ mlv="$1"
+ check_scm_mlv scm "$mlv"
+ check_dn_mlv dn1 "$mlv"
+ check_dn_mlv dn2 "$mlv"
+ check_dn_mlv dn3 "$mlv"
+ }
+
+ # Helper function, not a callback.
+ _check_om_mlvs() {
+ mlv="$1"
+ check_om_mlv om1 "$mlv"
+ check_om_mlv om2 "$mlv"
+ check_om_mlv om3 "$mlv"
+ }
+
+ setup() {
+ export OZONE_OM_PREPARE='true'
+ }
+
+ with_old_version() {
+ generate old1
+ validate old1
+ }
+
+ with_new_version_pre_finalized() {
+ _check_hdds_mlvs 2
+ _check_om_mlvs 0
+
+ validate old1
+ # HDDS-6261: overwrite the same keys intentionally
+ generate old1 --exclude create-volume-and-bucket
+
+ generate new1
+ validate new1
+
+ check_ec_is_disabled
+ }
+
+ with_old_version_downgraded() {
+ validate old1
+ validate new1
+
+ generate old2
+ validate old2
+
+ # HDDS-6261: overwrite the same keys again to trigger the precondition check
+ # that exists <= 1.1.0 OM
+ generate old1 --exclude create-volume-and-bucket
+ }
+
+ with_new_version_finalized() {
+ _check_hdds_mlvs 3
- _check_om_mlvs 2
++ _check_om_mlvs 3
+
+ validate old1
+ validate new1
+ validate old2
+
+ generate new2
+ validate new2
+
+ check_ec_is_enabled
+ }
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
index 65aae1e7ed,21edfe5d15..ad025d19f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
@@@ -26,8 -26,8 +26,7 @@@ import static org.apache.hadoop.hdds.pr
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
- import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 15eda5a4b9,3b9d3df975..f9245268f1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@@ -113,13 -116,9 +120,15 @@@ public class TestOzoneConfigurationFiel
ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_FLUSH_PARAM,
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
OMConfigKeys.OZONE_OM_HA_PREFIX,
+ OMConfigKeys.OZONE_OM_TRANSPORT_CLASS,
- OMConfigKeys.OZONE_OM_GRPC_PORT_KEY
++ OMConfigKeys.OZONE_OM_GRPC_PORT_KEY,
// TODO HDDS-2856
+ OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT,
+ OMConfigKeys.OZONE_RANGER_HTTPS_ADDRESS_KEY,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER,
+ OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD
));
}
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index fdbf17d9a4,e38c413e74..69977648a1
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@@ -102,9 -108,8 +109,9 @@@ public class TestOzoneShellHA
private static String testFilePathString;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
- private static OzoneShell ozoneShell = null;
- private static OzoneAdmin ozoneAdminShell = null;
- private static S3Shell s3Shell = null;
+ private OzoneShell ozoneShell = null;
+ private OzoneAdmin ozoneAdminShell = null;
++ private S3Shell s3Shell = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final ByteArrayOutputStream err = new ByteArrayOutputStream();
@@@ -170,6 -173,8 +175,9 @@@
@Before
public void setup() throws UnsupportedEncodingException {
+ ozoneShell = new OzoneShell();
+ ozoneAdminShell = new OzoneAdmin();
++ s3Shell = new S3Shell();
System.setOut(new PrintStream(out, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(err, false, DEFAULT_ENCODING));
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index deb77db639,d91d8519a9..c6a6745d5f
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -411,21 -356,7 +411,22 @@@ enum Status
LAYOUT_FEATURE_FINALIZATION_FAILED = 72;
PREPARE_FAILED = 73;
NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74;
+ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION = 75;
+
- TENANT_NOT_FOUND = 75;
- TENANT_ALREADY_EXISTS = 76;
- INVALID_TENANT_ID = 77;
++ TENANT_NOT_FOUND = 76;
++ TENANT_ALREADY_EXISTS = 77;
++ INVALID_TENANT_ID = 78;
+
- ACCESS_ID_NOT_FOUND = 78;
- TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 79;
- INVALID_TENANT_USERNAME = 80;
- INVALID_ACCESS_ID = 81;
- TENANT_AUTHORIZER_ERROR = 82;
++ ACCESS_ID_NOT_FOUND = 79;
++ TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 80;
++ INVALID_TENANT_USERNAME = 81;
++ INVALID_ACCESS_ID = 82;
++ TENANT_AUTHORIZER_ERROR = 83;
+
- VOLUME_IS_REFERENCED = 83;
- TENANT_NOT_EMPTY = 84;
++ VOLUME_IS_REFERENCED = 84;
++ TENANT_NOT_EMPTY = 85;
+
- FEATURE_NOT_ENABLED = 85;
++ FEATURE_NOT_ENABLED = 86;
}
/**
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d2848748a1,4e051875d7..ef2e8d70e9
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -67,10 -69,8 +69,11 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneManagerVersion;
+ import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
@@@ -127,12 -124,9 +130,13 @@@ import org.apache.hadoop.ozone.om.helpe
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.TenantStateList;
+import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
+import org.apache.hadoop.ozone.om.helpers.TenantUserList;
+ import org.apache.hadoop.ozone.om.lock.OzoneLockProvider;
import org.apache.hadoop.ozone.om.protocol.OMInterServiceProtocol;
import org.apache.hadoop.ozone.om.protocol.OMConfiguration;
import org.apache.hadoop.ozone.om.protocolPB.OMInterServiceProtocolClientSideImpl;
@@@ -300,6 -301,7 +314,7 @@@ public final class OzoneManager extend
private final Text omRpcAddressTxt;
private OzoneConfiguration configuration;
private RPC.Server omRpcServer;
- private GrpcOzoneManagerServer omS3gGrpcServer;
++ private GrpcOzoneManagerServer omS3gGrpcServer;
private InetSocketAddress omRpcAddress;
private String omId;
@@@ -339,7 -340,9 +354,9 @@@
private JvmPauseMonitor jvmPauseMonitor;
private final SecurityConfig secConfig;
private S3SecretManager s3SecretManager;
+ private final boolean isOmGrpcServerEnabled;
private volatile boolean isOmRpcServerRunning = false;
- private volatile boolean isOmGrpcServerRunning = false;
++ private volatile boolean isOmGrpcServerRunning = false;
private String omComponent;
private OzoneManagerProtocolServerSideTranslatorPB omServerProtocol;
@@@ -3910,21 -3741,20 +4057,21 @@@
throws IOException {
Pair<String, String> resolved;
- try {
- if (isAclEnabled) {
- InetAddress remoteIp = Server.getRemoteIp();
- resolved = resolveBucketLink(requested, new HashSet<>(),
- getRemoteUser(),
- remoteIp,
- remoteIp != null ? remoteIp.getHostName() :
- omRpcAddress.getHostName());
- } else {
- resolved = resolveBucketLink(requested, new HashSet<>(),
- null, null, null);
+ if (isAclEnabled) {
- UserGroupInformation ugi = Server.getRemoteUser();
++ UserGroupInformation ugi = getRemoteUser();
+ if (getS3Auth() != null) {
+ ugi = UserGroupInformation.createRemoteUser(
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId()));
}
- } catch (Throwable t) {
- throw t;
+ InetAddress remoteIp = Server.getRemoteIp();
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ ugi,
+ remoteIp,
+ remoteIp != null ? remoteIp.getHostName() :
+ omRpcAddress.getHostName());
+ } else {
+ resolved = resolveBucketLink(requested, new HashSet<>(),
+ null, null, null);
}
return new ResolvedBucket(requested, resolved);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 60a888e446,7b1f4185b6..4302327cf3
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@@ -31,8 -31,13 +31,14 @@@ public enum OMLayoutFeature implements
////////////////////////////// //////////////////////////////
INITIAL_VERSION(0, "Initial Layout Version"),
- // TODO: Make this 2 after bringing in EC
- MULTITENANCY_SCHEMA(1, "Multi-Tenancy Schema");
+ ERASURE_CODED_STORAGE_SUPPORT(1, "Ozone version with built in support for"
+ + " Erasure Coded block data storage."),
+
+ BUCKET_LAYOUT_SUPPORT(2, "Ozone version supporting bucket " +
+ "layouts and introducing the FILE_SYSTEM_OPTIMIZED and OBJECT_STORE " +
- "bucket layout types.");
++ "bucket layout types."),
+
++ MULTITENANCY_SCHEMA(3, "Multi-Tenancy Schema");
/////////////////////////////// /////////////////////////////
// Example OM Layout Feature with Actions
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 4dffc39e22,ecec94592d..94b9b6d56c
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -48,8 -48,11 +51,12 @@@ import org.apache.hadoop.ozone.om.ratis
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+ import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+ import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationCondition;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
@@@ -78,15 -80,12 +86,17 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index 891c48ca96,e9ddc08815..ca8d32b48d
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@@ -71,9 -66,7 +66,7 @@@ public class OzoneClientProducer
@Produces
public synchronized OzoneClient createClient() throws WebApplicationException,
IOException {
- if (client == null) {
- client = createOzoneClient();
- }
- client = getClient(ozoneConfiguration);
++ client = getClient(ozoneConfiguration);
return client;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org