You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sz...@apache.org on 2022/05/06 15:45:05 UTC
[ozone] branch HDDS-4454 updated (7c5356b4e6 -> 74ce5363bd)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a change to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 7c5356b4e6 HDDS-6592. [Ozone-Streaming] Fix ContainerStateMachine#applyTransaction assert error (#3315)
omit 1069b64616 HDDS-5666. Add option to createKey via streaming api in Freon (#2574)
omit 30c9cc890a HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at the end of the stream. (#3229)
omit 9546d4c641 HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput. (#3195)
omit bc152e1234 HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters. (#3207)
omit b9d36ba86c HDDS-6461. Update Ratis version to 2.3.0-da5d868-SNAPSHOT. (#3205)
omit b29e7fa9b3 HDDS-6388. [Ozone-Streaming] Streaming write support both pipeline model and star model (#3145)
omit 7a218caf8d HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
omit f64365b77d HDDS-6229. [Ozone-Streaming] Data Channel abstraction on datanode (#3023)
omit 1bb85c74f5 HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE (#3060)
omit 03ec7f1533 HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support FlushDelay. (#3002)
omit 083002e123 HDDS-6298. Add XceiverServerRatis stream config (#3070)
omit d290fba027 HDDS-6138.[Ozone-Streaming] Define a limit on the size of the retry bufferList. (#2946)
omit 8423a6d8f6 HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT version (#3059)
omit f9bcfb0e91 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
omit 7107ae0612 HDDS-6139. [Ozone-Streaming] Fix incorrect computation of totalAckDataLength. (#2978)
omit e831fa697e HDDS-6130. [Ozone-Streaming] When releaseBuffers will get “Couldn 't find the required future” (#2939)
omit 546987742b HDDS-6039. Define a minimum packet size during streaming writes. (#2883)
omit 3722393d04 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. (#2866)
omit 718b9574f6 HDDS-5743. [Ozone-Streaming] Add option to write files via streaming api in ofs and o3fs. (#2770)
omit a8d7669207 HDDS-5879. [Ozone-Streaming] OzoneBucket add the createMultipartStreamKey method (#2760)
omit 6bf06d1a93 HDDS-5961. [Ozone-Streaming] update the usage space of Containers in the stream write (#2833)
omit 8b31829876 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream config (#2841)
omit e5a281d8c6 HDDS-5763. Provide an Executor for each LocalStream in ContainerStateMachine (#2782)
omit 05bc933b0a HDDS-5895. [Ozone-Streaming] Make raft.server.data-stream.client.pool.size configurable (#2766)
omit ea34f61832 HDDS-5674.[Ozone-Streaming] Handle client retries on exception (#2701)
omit 7993ae5f43 HDDS-5849. [Ozone-Streaming]Write exceptions occur after checksum is enabled (#2729)
omit af5a9f5c5f HDDS-5486. [Ozone-Streaming] Streaming supports writing in Pipline mode (#2682)
omit cc1daa9c00 HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
omit 4464d8aeec HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to ByteBufferStreamOutput (#2603)
omit c1e4e8ee89 HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer to avoid buffer copying (#2557)
omit 3e37724d78 HDDS-5488. [Ozone-Streaming] Add a new BlockOutputStream/KeyOutputStream to support streaming api (#2495)
omit c5545081ed HDDS-5480. [Ozone-Streaming] Client and server should support stream setup. (#2452)
omit 8efe1952e5 HDDS-5481. Fix stream() and link() method in ContainerStateMachine. (#2451)
omit fa4e80ffb3 HDDS-5452. Add link method to ContainerStateMachine for Ratis streaming (#2422)
omit b7f2cc86de HDDS-5366. [Ozone-Streaming] Implement stream method to ContainerStateMachine. (#2358). Contributed by mingchao zhao
add cf8eb336ec HDDS-5210. Create GRPC service definition for existing OM protocol (#2375)
add 3b4e2f58b3 Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
add 1bce6dfd11 HDDS-5213. Create and a specific OmTransportFactory for GrpcOmTransport (#2485)
add 46575efe21 HDDS-5211. Start OM Grpc server as part of the OM bootstrap (#2601)
add a115ebfadc HDDS-5212. Create implementation of OmTransport which uses Grpc (#2580)
add 1b891fb64f HDDS-5612. Create Ozone s3 authentication token and client authentication transport (#2655)
add b0f41d67da HDDS-5780. Support s3 authentication on a per request basis (#2721)
add 1ffbeddcec HDDS 5781. Enable ACLs and support for all s3 file operations. (#2739)
add c9442d2507 HDDS-5778. Client side unit tests (#2810)
add 4d0bf19895 Commit for merge requested in PR #2945 to merge Apache Ozone master branch with HDDS-5881 changes INTO HDDS-4400-s3-performance feature branch. Merge allows s3 gateway gRPC to work with s3 gateway persistent connection changes made in HDDS-5881 found in master branch.
add abb3637a08 Synchronization added to getOzoneConfiguration for OzoneClientProducer to satisfy CI findbugs.
add b88a45e994 HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTransport support (#3074)
add 413d4aade2 HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)
add 0b0c586fec HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)
add 3f585ab05d HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)
add d5ade578f4 Revert "HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)"
add d87a7461c1 Revert "HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)"
add d57616e165 Revert "HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)"
add 2522127b1c Revert "HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTransport support (#3074)"
add f7b2dbd8f2 Necessary due to CI workflow issue, HDDS-6239, impacting feature branch green builds. Merging with Master branch impacts HDDS-4440 PRs in progress. This merge also requires partial changes that were initially in PR for HDDS-5545 to be included. Namely, the patch to allow switching between OmTransports, GrpcOmTransport and Hadoop3OmTransport through ozone configuration in the OmConfigKeys.
add e3259164ca Trigger Build
add 666525e386 Fix integration test for added configuation field for selecting OmTransport for s3 gateway - TestOzoneConfigurationFields (added config key not in xml).
add 0de099dbde Merge remote-tracking branch 'neils-dev/merge-master-s3g' into HDDS-4440-s3-performance
add 76e238118d HDDS-5544. Update GRPC OmTransport implementation for HA (#2901)
add 47e457697e HDDS-5545. Enable TLS for GRPC OmTransport implementation (#2945)
add 062890ee4b HDDS-5648. Track CI workflow tests temporarily disabled for feature branch (#3257)
add 75f5501602 HDDS-6481. Add metrics for S3 gateway (#3221)
add b21a613146 HDDS-6473. Empty data in response for V0 GetSmallFile request (#3230)
add d35da4091d HDDS-6098. Add a metric to track amount of data committed. (#2913)
add e07c66ca27 HDDS-6425. OmMetadataManagerImpl#isBucketEmpty does not work on FSO buckets. (#3227)
add 605746617b HDDS-6497. S3 Gateway throws exception when checking jmx or prom pages from browser (#3228)
add ff9f523899 HDDS-6509. Checkstyle: Enable setterCanReturnItsClass in HiddenField (#3237)
add a8ef728055 HDDS-6499. Cleanup OMRequest class constructors. (#3248)
add 682be322fa HDDS-6367. ContainerBalancer shows incorrect iteration result sometimes (#3129)
add d1e39315f0 HDDS-6502. Blocks of old key versions are not deleted on key deletion (#3232)
add 0c4a01766b HDDS-6516. Shade Kotlin for Ozone Filesystem (#3246)
add 0ec6005ce7 HDDS-6429. getContainerReplicas should respect client version (#3231)
add 0bf2090078 HDDS-6505. TestContainerStateManagerIntegration consistently fails (#3234)
add adecbd1f92 HDDS-6362. Add aggregate metrics to ContainerBalancerMetrics (#3239)
add 0262c8ac3a HDDS-6466. Drop S3 Gateway bucket browser feature (#3241)
add 2eabcc5bd5 HDDS-5963. Implement ListBlock command (#3212)
add d93fbdfa64 HDDS-6397. Implement ContainerBalancer as an SCMService (#3153)
add 1d40e55e91 HDDS-6538. Update Spring to 5.2.20 to fix CVE-2022-22965 aka Spring4shell. (#3261)
add a6b09e777e HDDS-6456 Update RocksDB Version to 7.0.4 (#3199)
add c23aae0d93 HDDS-6435. Add read lock waiting and held time metrics (#3180)
add 68270e46bf HDDS-6414. Ozone doesn’t reclaim space after deletion of volume via ofs (#3175)
add 15a09aa127 HDDS-6523. Refine OMClientRequest Classes to handle bucket layout better. (#3265)
add 0f04fa7b17 HDDS-6537. Deprecate the renameKeys API (#3260)
add aa68b9abed HDDS-5119. Recon file count by size page has incorrect data when keys are deleted (#3269)
add 805b5e1d3b HDDS-6545. OzoneManagerUtils#getBucketLayout should not suppress bucket key lookup errors. (#3277)
add 76f809f3ec HDDS-6436. Add write lock waiting and held time metrics (#3271)
add 473627b8f5 HDDS-6554. Have the datanode heartbeat include queued command counts (#3275)
add e81ab0a3f7 HDDS-6564. DirectoryDeletionService should use getFileTable to handle bucket delete (#3284)
add 55d5ecbda2 HDDS-6552. Bump Jackson Databind (#3273)
add 64d67668d2 HDDS-5862. Datanode to Recon heartbeat interval independently configurable (#3245)
add a2e169393c Merge of master into feature branch to sync the feature branch to the current master. Resolved conflicts affecting feature versioning PR#3155, propagating s3 exception stack trace HDDS-6257 PR#3066 and performFailover PR#3160.
add 55d2e06cf2 Fix checkstyle
add de0b1f9b81 HDDS-6528. Adding Unit-Test cases for S3-Gateway Bucket-Endpoint Metrics. (#3263)
add 4c1ba2b998 HDDS-6535. Ozone TokenRenewer not found (#3259)
add 141ced1196 HDDS-6579. Add commandline argument for Container Info command. (#3306)
add 22cdd95a9e HDDS-6141. terminate om if statemachine is shut down by ratis (#2953)
add c583b3e7c6 HDDS-6162. Limit OM DoubleBuffer pending requests to avoid taking too much memory (#3024)
add d8a21a2bdb HDDS-6593. Bump node to v16.14.2 for Recon (#3316)
add 0b4af02bfb HDDS-6591. Javadoc - update StorageContinerLocationProtocol for change of qualified class name (#3314)
add 2851ea53f0 HDDS-5339. Document OM bootstrap and decommission process (#3299)
add 8f98af911c HDDS-6547. om snapshot failed file not deleted. (#3268)
add fe49f69a33 HDDS-6346. OmFailoverProxyProvider cleanup in loadConfigs for legacy service ids code (#3313)
add d5eb702c68 HDDS-6587. SCM Web UI is not displaying HEALTHY_READONLY header in node status (#3311)
add a0684e26c8 HDDS-6574. Set owner of buckets created via S3 Gateway to actual user rather than `s3g`; print LinkBucket owner field on the client (#3298)
add 3fd7dc6c77 HDDS-6525. Add audit log for S3Gateway (#3252)
add 0b6f46714f HDDS-6562. Exclude specific operations in Audit log (#3289)
add 172947815b HDDS-6532. Enable READ audit log by default (#3255)
add 204584b395 HDDS-6515. Intermittent failure in TestRootedOzoneFileSystem (#3322)
add cec910568c HDDS-5003. Introduce EC replication type (#2064)
add a51efc541b HDDS-4954. Add replicaIndex to the RPC protocols (#2055)
add 23fe4f5e56 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add a915dbfdd5 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add c050b8f67e HDDS-5011. Introduce ECReplicationConfig and Java based ReplicationConfig implementation (#2068)
add aeedef8c7b Merge branch 'master' into the EC branch
add 1d5c593319 HDDS-5145. Extend Pipline/ReplicationConfig refactor with ECReplicationConfig (#2206)
add 132911c504 HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
add 9c4008d540 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add ad790b660e HDDS-4883. Persist replicationIndex on datanode side (#2069)
add c960ec7c5a Merge remote-tracking branch 'origin/HDDS-3816-ec' into HDDS-3816-ec
add f904539c0f Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add fbea53d4f5 Merge branch 'master' into EC branch
add b6f960daaa HDDS-5313. ContainerInfo should use ReplicationConfig (#2320)
add 9e0a2fc4e2 HDDS-5247. EC: Create ECReplicationConfig on client side based on input string (#2331)
add 5e42c31933 Merge branch 'master' into EC branch.
add 27f57b7d3c Fixed conflict issue as part of merge branch 'master' into EC branch.
add f53a93e6a9 HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
add 206c9358a5 HDDS-4892. EC: Implement basic EC pipeline provider (#2353)
add 3a94742765 HDDS-5375. EC: Extend PipelineManager.createPipeline API to support excluded nodes (#2378)
add 4115827ff9 HDDS-4940 : EC: Implement the ECKeyOutputStream which should handle the EC mode writes. (#2335)
add cd781b9785 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
add a0f6881a28 Merge branch 'master' into ec-merge-master
add 0c5c859b99 Fix issues caused by merging master into branch
add 2ef364f47b HDDS-5509. EC: Add missing break in switch statement when requesting EC blocks (#2468)
add a64945bd7c HDDS-5325. EC: Add configuration to set an EC container placement policy (#2483)
add 942a23f14e Merge remote-tracking branch 'origin/master' into merge-master-0803
add f3ce6c7801 HDDS-5510: allocateContainer should handle ec replication config (#2490)
add 6eba2ff17d HDDS-5543: OMKeyRequest#createFileInfo should handle ECReplicationConfig (#2501)
add 20851dc199 HDDS-5561. EC: ECReplicationConfig should be immutable (#2516)
add d2dca9b079 HDDS-5562. EC: ContainerPlacementPolicyFactory#getPolicyInternal() should not be public. (#2517)
add 94814e5dba HDDS-5425: Client side pipeline cache key should use host and port combination (#2510)
add 0e6e00e97e Merge branch 'master' into ec-branch
add db35f8c003 HDDS-4942. EC: Implement ECBlockInputStream to read a single EC BlockGroup (#2507)
add d6dc278195 EC: Make ECReplicationConfig stored as bucket level attributes. (#2401)
add 823b505ab5 HDDS-5326. EC: Create a new as many racks as possible placement policy for EC (#2497)
add 90c4d9286d HDDS-5470 : EC: Add padding and generate parity if the last stripe is not full (#2455)
add 2df6c9caf5 Merge branch 'master' into ec branch
add e08d5009ab Removed SCMPipelineManager.java which should have been removed via HDDS-5554, but was somehow missing on an earlier master to ec branch merge.
add 6b17c7570d HDDS-5745. EC: Pipeline builder should copy replica indexes from original pipeline (#2642)
add 1d3423de76 HDDS-5477. EC: commit key should consolidate and create one keyLocationInfo per blockGrp (#2648)
add ebdd1f5e3d HDDS-5682: EC: Provide replication config option from CLI when creating bucket. (#2639)
add b55107fe76 HDDS-5550. EC: Adapt KeyInputStream to read EC Blocks (#2653)
add 0a0e62e002 EC: Add Codec and Stripe Size to ECReplicationConfig (#2674)
add fc17518932 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
add 5cf251654e HDDS-5741. EC: Remove hard coded chunksize and get from from ReplicationConfig (#2687)
add 831b3faa27 HDDS-5833: EC: Fix TestRootedOzoneFileSystem.testBucketDefaultsShouldBeInheritedToFileForEC failure in branch (#2718)
add 7a67f63429 HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
add a16ee9d2fb Merge branch 'master' into HDDS-3816-ec
add aee577c171 HDDS-5825: EC: ECKeyOutputStream#close fails if we write the partial chunk (#2720)
add f478b96360 HDDS-5832. EC: ECKeyOutputStream persists blocks in random order (#2717)
add d37fe6901b HDDS-5552. EC: Implement seek on ECBlockInputStream (#2723)
add 38e4cca273 HDDS-5755 EC: Refactor ECBlockOutputStreamEntry to accommodate all block group related ECBlockOuputStreams. (#2702)
add 914ce67af3 HDDS-5364: EC: Adopt EC related utility from Hadoop source repository (#2733)
add f5dbe00309 Hdds 5491: EC: Write should handle node failures. (#2767)
add c9fa9bdf0f HDDS-5874 EC: Integrate the Codec changes into EC Streams. (#2777)
add 818118a7ad HDDS-5948: EC: Fix the compile issue in TestOzoneECClient (Due to concurrent commits) (#2812)
add e82682fe8b HDDS-5551. EC: Implement an Input Stream to reconstruct EC blocks on demand (#2797)
add f85ac376e6 HDDS-5950. EC: Implement seek on ECBlockReconstructedStripeInputStream (#2818)
add 438799ba75 HDDS-5953: EC: Review the current flush API and clean up (#2828)
add 52f05fc349 HDDS-5951. EC: ECBlockReconstructedStripeInputStream should handle block read failures and continue reading (#2831)
add 67043109c0 Merging master into EC branch
add d5ac6738b7 HDDS-6012: EC: Fix TestOzoneShellHA failures post master merge with EC branch
add 9a7025a39b HDDS-6035: EC: Fix TestOmMetrics in merge branch
add a0349bd9a9 HDDS-6029: EC: Fix the replication config handling in OMDirectoryCreateRequest#dirKeyInfoBuilderNoACL (#1)
add 685442c6b8 HDDS-6001. EC: Create ECBlockReconstructedInputStream to wrap ECBlockReconstructedStripeInputStream (#2848)
add 93fadc9789 HDDS-5979: EC: Change CLI bucket default replication option name to -type (#2873)
add 6e7d0dc206 HDDS-6009. EC: Optimize ECBlockReconstructedStripeInputStream where there are no missing data indexes (#2872)
add 1b9724d60a HDDS-5955: EC: Track the failed servers to add into the excludeList when invoking allocateBlock (#2849)
add 460aa9b89b HDDS-6020: EC: Provide set replicationConfig option to bucket (#2880)
add a1d7292bcf HDDS-6010. EC: Create ECBlockInputStreamProxy to choose between reconstruction and normal reads (#2889)
add 73bddf36d0 Updates to address compile issues after merge
add 3007b5c68d HDDS-6081. EC: Fix ReplicationConfig related test failures that happened due to merging HDDS-5997 to the EC branch
add c6604833d1 HDDS-5952. EC: ECBlockReconstructedStripeInputStream should read from blocks in parallel (#2899)
add 86f735418a HDDS-6036: EC: handleStripeFailure should retry (#2910)
add f8fd2ff66b HDDS-6065: EC: Provide CLI option to reset the bucket replication config (#2927)
add ba192b1de1 HDDS-6132: EC: HandleStripeFailure should not release the cachebuffers. (#2940)
add 3eb55db014 HDDS-5949. EC: Create reusable buffer pool shared by all EC input and output streams (#2929)
add 5856411168 HDDS-6179. EC: Provide toString in ECReplicationConfig (#2986)
add 07b2338525 HDDS-6180. EC: Pipeline creator should ignore creating pipelines for ZERO factor (#2985)
add aaf6f17045 HDDS-6021: EC: Client side exclude nodes list should expire after certain time period or based on the list size. (#2973)
add d0467bac1d HDDS-6220. EC: Introduce a gRPC client implementation for EC with really async WriteChunk and PutBlock. (#3016)
add b40ab3f97b HDDS-6172: EC: Document the Ozone EC (#3006)
add 78ccdd540c HDDS-6131: EC: Replication config from bucket should be refreshed in o3fs (#3008)
add 6b5c01c843 HDDS-6184. EC: put command should create EC key if bucket is EC (#2990)
add 4d660c4d79 HDDS-6233. EC: Bucket does not display correct EC replication details (#3025)
add 2e60a4f8cb Merge master to EC Branch
add 58f8e30538 HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist (#3029)
add 78ca097eac HDDS-6083: EC: Fix flakyness of tests around nodefailures (#3021)
add dfa8141937 HDDS-6237: EC: putBlock should pass close flag true on end of block group/close file. (#3026)
add ee630e2995 HDDS-6240. EC: Container Info command with json switch fails for EC containers (#3033)
add 8e52561eab HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3037)
add ddcfc521af HDDS-6252. EC: Datanode Chunk Validator fails on encountering EC pipeline (#3035)
add ca6e4c3599 HDDS-6255. EC: Replication Manager should skip EC Containers (#3038)
add 5d2fa94949 HDDS-6256. EC: Parity blocks are incorrectly padded with zeros to the chunk size (#3043)
add 9a0f812b3b HDDS-6231. EC: Create EC acceptance test environment and some basic tests (#3030)
add 722c444aea HDDS-6250. EC: Add replica index to the output in the container info command (#3041)
add 29c69b109b HDDS-6272: EC: Recon UI failing because of EC replication factor (#3067)
add 22e34820cf HDDS-6258. EC: Read with stopped but not dead nodes gives IllegalStateException rather than InsufficientNodesException (#3048)
add c4aec7bb97 HDDS-6259. EC: Pipelines for closed containers should contain correct replica indexes (#3062)
add dafdb23255 HDDS-6295. EC: Fix unaligned stripe write failure due to length overflow. (#3076)
add 2d63cc4ec9 HDDS-6194. EC: Freon ockg support EC write (#2996)
add 79f60c4d5c HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC libraries (#3087)
add aebaff55a2 Merge master to EC branch HDDS-3816-ec
add b2a33b759e HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
add 4b9d38763a HDDS-6294. EC: Make cluster-wide EC configuration take effect (#3089)
add 550eadc970 HDDS-6308. EC: Handle Replication Factor to consider EC Config in Recon UI (#3078)
add 6f64eb77d5 HDDS-6342. EC: Fix large write with multiple stripes upon stripe failure. (#3108)
add 60a41bc85e HDDS-6249. EC: Fix todo items in TestECKeyOutputStream (#3102)
add 1d23619164 HDDS-6188: EC: Adapt java side native coder classes from hadoop. (#3046)
add 387fb1a2c1 HDDS-6232. EC: Update help strings for replication config (#3113)
add 0a83e7018e HDDS-6341: EC: Fix the race condition in TestECBlockReconstructedStripeInputStream. (#3111)
add aac07ef790 HDDS-6347. EC: Freon randomKeys EC key support (#3112)
add 6b8d935f73 HDDS-6319. EC: Fix read big file failure with EC policy 10+4. (#3094)
add b9adbcd89a HDDS-6348: EC: PartialStripe failure handling logic is writing padding bytes also to DNs (#3124)
add b90f00192d HDDS-6372. EC: Do not throw NotImplementedException in flush() (#3133)
add 78ca327e77 HDDS-6358. EC: Refactor ECKeyOutputStream#write() (#3120)
add 9e3cf98a30 HDDS-6279. EC: Calculate EC replication correctly when updating bucket usage (#3136)
add e675e3ba3d HDDS-6380. EC: Key Info command should not display legacy replication fields as they duplicate ReplicationConfig (#3141)
add 9ee6d72f31 HDDS-6373. EC: Exclude pipeline upon container close instead of exclude DNs. (#3137)
add 38409ea5a0 HDDS-6364. EC: Discard pre-allocated blocks to eliminate worthless retries. (#3127)
add 9680edf850 HDDS-6384. EC: Ensure EC container usage is updated correctly when handling reports (#3147)
add 58eac935cd HDDS-6185: EC: EC keys can't be created via S3 interfaces (#3150)
add 8cff9fd2ad HDDS-6368. EC: Fix broken future chain and cleanup unnecessary validation function. (#3128)
add 8776430812 HDDS-6382. EC: ListPipelines command should consider EC Containers (#3161)
add 28482bf2b9 HDDS-6422. EC: Fix too many idle threads during reconstruct read. (#3168)
add 074203fc0a HDDS-6437. EC: Avoid allocating buffers in EC Reconstruction Streams until first read (#3181)
add 9675863705 HDDS-6413. EC: OzoneManagerRequestHandler needs to handle ECReplicationConfig (#3198)
add 06f1eb5ad6 HDDS-6248. EC: Container list command should allow filtering of EC containers (#3179)
add 03ff3f9592 HDDS-6445. EC: Fix allocateBlock failure due to inaccurate excludedNodes check. (#3190)
add af48c537a9 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
add 217c1911b2 EC: Check isFullCell inside handleDataWrite (#3202)
add 9cda6a1a06 HDDS-6460. EC: Adjust requested size by EC DataNum in WritableECContainerProvider (#3203)
add 04c62c53f2 HDDS-6411. EC: OmMultipartKeyInfo needs to handle ECReplicationConfig (#3210)
add fc6ac0c48c HDDS-6448. EC: Reconstructed Input Streams should free resources after reading to end of block (#3197)
add f386662a37 HDDS-6381. EC: Improve exception message in ByteBufferEncodingState (#3143)
add 9ec90f5820 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec-merge-master
add c55f30edf0 HDDS-6412: EC: Handle ECReplicationConfig in initiateMultipartUpload API (#3184)
add 16c3e99eae HDDS-6478. EC: Fix ISA-l load hadoop native lib UnsatisfiedLinkError (#3218)
add 143315c89b HDDS-6409. EC: OzoneMultipartUpload needs to handle ECReplicationConfig (#3279)
add d9529b32ea HDDS-6434: EC: Overwriting an EC key with a Ratis key fails (#3274)
add df96b86c8d HDDS-6557. EC: Execute S3 acceptance tests with EC (#3281)
add e9b6215210 Merge master to EC Branch HDDS-3816-ec
add e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
add 6c6b740a47 HDDS-6209. EC: [Forward compatibility issue] New client to older server could fail due to the unavailability for client default replication config (#3267)
add 70b547be23 HDDS-5909. EC: Onboard EC into upgrade framework (#3262)
add cb0a84b27c Merge master to EC branch HDDS-3816-ec
add 96669856ad Merge EC branch HDDS-3816-ec to master
add 681d0a8f5e HDDS-6471. Only repeat flaky tests (#3247)
add dc9da7d405 HDDS-6606. Rat check failure ignored for ozone-annotation-processing (#3321)
add be5888b907 HDDS-6580. Introduce KEY_PATH_LOCK under OzoneManagerLock.Resource (#3308)
add 4345fb1c58 HDDS-6189. Intermittent failure in TestDirectoryDeletingServiceWithFSO (#3328)
add 03b0a61958 HDDS-6583. Introduce lock strategy pattern implementations based on configuration flags (#3310)
add cb2754fd9c HDDS-6555. Container Deletion should not depend on usedBytes being zero (#3276)
add b077cc6123 HDDS-6561. Modify default bucket layout to LEGACY. (#3296)
add 9ac8fbce70 HDDS-6614. EC: Fix Datanode block file INCONSISTENCY during heavy load. (#3323)
add 2876c54b8d HDDS-6405. Selective checks: run rat for readme change (#3156)
add 497c7a40b8 HDDS-6617. Skip compile(11) for non-java changes (#3324)
add b8388b096a HDDS-6639. Remove flaky tag for testOMProxyProviderFailoverOnConnectionFailure (#3343)
add b78adc4c71 HDDS-6620. Gather AuditLog params for S3g (#3325)
add de9fcca1c3 HDDS-6522. Fix TestOzoneManagerHAWithData#testTwoOMNodesDown (#3341)
add 2ea9e78bf1 Merge remote-tracking branch 'origin/master' into HDDS-4440-merge
add edec164302 HDDS-6643. Minor fixes for OM gRPC feature (#3347)
add db7f691855 HDDS-6651. Restore S3 Gateway config key check in TestOzoneConfigurationFields (#3354)
add db778fce96 HDDS-6457. Add freon command to run r/w mix workload using FS (#3266)
add c6314b987c HDDS-6512. Install Hugo during build if needed (#3287)
add f575e238f8 HDDS-6553. Incorrect timeout when checking balancer iteration result. (#3272)
add 9d771eb6ed HDDS-6647. Tag some known flaky tests (#3349)
add b4e8e7ac8d HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
add 73d4876dfe Revert "HDDS-6579. Add commandline argument for Container Info command. (#3306)" (#3336)
add f49a691b41 HDDS-6626. Exclude Hadoop Thirdparty from shaded Ozone FS jars (#3330)
add 4476a5be19 HDDS-6632. ReplicationManagerReport should handle unknown metric in protobuf (#3335)
add d9a623ad72 HDDS-6598. Add a BackgroundPipelineScrubber to scrub all pipelines. (#3337)
add 1944b2cda9 HDDS-6624. Make ECBlockReconstructedStripeInputStream available for datanodes (#3353)
add 240002559e Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
add 124723a4b2 HDDS-6661. TestBackgroundPipelineScrubber.testRun() fails intermittently (#3359)
add c251258832 HDDS-6658. BackgroundPipelineCreator does not always stop quickly (#3357)
add 81b6bbad14 HDDS-6641. datanode usageinfo CLI should provide JSON output option (#3356)
add 59f60089bb HDDS-6653. Small fixes in PipelineStateMap (#3355)
add d2ac33627a HDDS-6567. Store datanode command queue counts from heartbeat in DatanodeInfo in SCM (#3329)
add d53cb2c1b7 HDDS-6529. Adding Unit-Test cases for S3-Gateway Object-Endpoint Metrics (#3302)
add ba9f3425ba HDDS-6684. Move unit tests out of integration (#3375)
add deb88f35ec HDDS-6586: EC: Implement the EC Reconstruction Command with necessary information (#3345)
add e70bfd0689 Revert "HDDS-5740. Enable ratis by default for SCM. (#2637)" (#3362)
add 28170f2672 HDDS-6685. Follower OM crashed when validating S3 auth info. (#3376)
add c33f6a58d8 HDDS-6640. Node.isAncestor might return incorrect result (#3342)
add c6cd84cc70 HDDS-6377. Redundant loop while doing triggerHeartbeat in DatanodeStateMachine (#3139)
add 71ce98081d HDDS-6673. Reduce OzoneFS test combinations in ozonesecure (#3368)
add c99fd20da9 HDDS-6589. Add a new replication manager and change the existing one to legacy (#3352)
add a3f5021b23 HDDS-6669. Exercise OM in ozonescripts environment (#3367)
add 9ff16ac551 HDDS-6655. Use injected clocks in PipelineManagerImpl and BackgroundPipelineScrubber to ease testing (#3363)
add f315500c73 HDDS-6551. Introduce StatefulService in scm (#3307)
new 553346e7d2 HDDS-5366. [Ozone-Streaming] Implement stream method to ContainerStateMachine. (#2358). Contributed by mingchao zhao
new 133557f3c0 HDDS-5452. Add link method to ContainerStateMachine for Ratis streaming (#2422)
new d32ae35318 HDDS-5481. Fix stream() and link() method in ContainerStateMachine. (#2451)
new 14856e8808 HDDS-5480. [Ozone-Streaming] Client and server should support stream setup. (#2452)
new 3641548d56 HDDS-5488. [Ozone-Streaming] Add a new BlockOutputStream/KeyOutputStream to support streaming api (#2495)
new 0790843af0 HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer to avoid buffer copying (#2557)
new 3479f54b69 HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to ByteBufferStreamOutput (#2603)
new 7b713b2b5f HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
new 7663ee5766 HDDS-5486. [Ozone-Streaming] Streaming supports writing in Pipline mode (#2682)
new 247c4f2324 HDDS-5849. [Ozone-Streaming]Write exceptions occur after checksum is enabled (#2729)
new 5049243ac1 HDDS-5674.[Ozone-Streaming] Handle client retries on exception (#2701)
new 2c1365e1b4 HDDS-5895. [Ozone-Streaming] Make raft.server.data-stream.client.pool.size configurable (#2766)
new 0f5a55a62d HDDS-5763. Provide an Executor for each LocalStream in ContainerStateMachine (#2782)
new 2a629f82c9 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream config (#2841)
new bb41750fdd HDDS-5961. [Ozone-Streaming] update the usage space of Containers in the stream write (#2833)
new 22d0553467 HDDS-5879. [Ozone-Streaming] OzoneBucket add the createMultipartStreamKey method (#2760)
new e1d9db9899 HDDS-5743. [Ozone-Streaming] Add option to write files via streaming api in ofs and o3fs. (#2770)
new 4771a50e6a HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. (#2866)
new ac9815d47c HDDS-6039. Define a minimum packet size during streaming writes. (#2883)
new d9d81f5b97 HDDS-6130. [Ozone-Streaming] When releaseBuffers will get “Couldn 't find the required future” (#2939)
new 6925126ddc HDDS-6139. [Ozone-Streaming] Fix incorrect computation of totalAckDataLength. (#2978)
new 59eb086e9a HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
new a65a63ee7d HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT version (#3059)
new 1bbc76f47a HDDS-6138.[Ozone-Streaming] Define a limit on the size of the retry bufferList. (#2946)
new 4fa81b239c HDDS-6298. Add XceiverServerRatis stream config (#3070)
new 32613fba33 HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support FlushDelay. (#3002)
new c13128155b HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE (#3060)
new b281043220 HDDS-6229. [Ozone-Streaming] Data Channel abstraction on datanode (#3023)
new c5a9686ee6 HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
new 0d9b3e0e27 HDDS-6388. [Ozone-Streaming] Streaming write support both pipeline model and star model (#3145)
new ae91a3f869 HDDS-6461. Update Ratis version to 2.3.0-da5d868-SNAPSHOT. (#3205)
new eeeddff8b9 HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters. (#3207)
new b23fbcf9bd HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput. (#3195)
new 246b0a6f2d HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at the end of the stream. (#3229)
new 74ce5363bd HDDS-5666. Add option to createKey via streaming api in Freon (#2574)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7c5356b4e6)
\
N -- N -- N refs/heads/HDDS-4454 (74ce5363bd)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 35 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 50 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 177 ++++
.../hdds/scm/storage/ExtendedInputStream.java | 91 ++
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 ++
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 ++
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 ++
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 +++++
.../client/io/ECBlockReconstructedInputStream.java | 243 +++++
.../io/ECBlockReconstructedStripeInputStream.java | 631 ++++++++++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io/package-info.java | 0
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 ++++++++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 ++
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++++++++
.../client/io/TestECBlockInputStreamProxy.java | 396 ++++++++
.../io/TestECBlockReconstructedInputStream.java | 377 +++++++
.../TestECBlockReconstructedStripeInputStream.java | 721 +++++++++++++
hadoop-hdds/common/pom.xml | 4 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 21 +
.../hdds/client/DefaultReplicationConfig.java | 134 +++
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 16 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 12 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 22 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 39 +-
.../ContainerCommandResponseBuilders.java | 13 +-
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +--
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 49 +-
.../common/src/main/resources/ozone-default.xml | 146 ++-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 ++
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +++-
.../hdds/scm/container/TestContainerInfo.java | 86 ++
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 ++
.../container/common/helpers/TestExcludeList.java | 67 ++
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 ++
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 54 +-
.../ozone/container/ContainerTestHelper.java | 28 +
.../common/src/test/resources/auditlog.properties | 2 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/statemachine/DatanodeStateMachine.java | 35 +-
.../common/statemachine/StateContext.java | 26 +
.../commandhandler/CommandDispatcher.java | 16 +
.../commandhandler/CommandHandler.java | 10 +
.../commandhandler/DeleteBlocksCommandHandler.java | 11 +
.../DeleteContainerCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../states/datanode/RunningDatanodeState.java | 8 +-
.../states/endpoint/HeartbeatEndpointTask.java | 21 +
.../states/endpoint/RegisterEndpointTask.java | 6 +-
.../server/ratis/ContainerStateMachine.java | 3 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 55 +-
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 10 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../replication/ReplicationSupervisor.java | 2 +-
.../commands/ReconstructECContainersCommand.java | 190 ++++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../container/common/impl/TestHddsDispatcher.java | 20 +
.../common/statemachine/TestStateContext.java | 36 +
.../states/endpoint/TestHeartbeatEndpointTask.java | 26 +-
.../container/keyvalue/TestKeyValueHandler.java | 7 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 ++
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++++
hadoop-hdds/docs/content/feature/OM-HA.md | 37 +
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
.../docs/content/start/StartFromDockerHub.md | 6 -
.../docs/content/start/StartFromDockerHub.zh.md | 5 -
hadoop-hdds/docs/pom.xml | 12 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 ++
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 ++
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +++
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 ++
.../rawcoder/AbstractNativeRawEncoder.java | 99 ++
.../rawcoder/ByteArrayDecodingState.java | 132 +++
.../rawcoder/ByteArrayEncodingState.java | 105 ++
.../rawcoder/ByteBufferDecodingState.java | 143 +++
.../rawcoder/ByteBufferEncodingState.java | 110 ++
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 ++
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 ++
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 ++
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 ++
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 ++
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 ++
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 ++
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 ++
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 ++
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 ++
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 ++++++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++++++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 ++
.../ozone/erasurecode/TestCodecRegistry.java | 159 +++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++++++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 ++
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 ++
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +++
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 ++
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 51 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 14 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 28 +
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 20 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/balancer/ContainerBalancer.java | 502 +++++----
.../balancer/ContainerBalancerMetrics.java | 86 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../IllegalContainerBalancerStateException.java | 36 +-
...lidContainerBalancerConfigurationException.java | 37 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 ++++++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 ++-----
.../replication/ReplicationActivityStatus.java | 104 --
.../container/replication/ReplicationManager.java | 450 +++++++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 5 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 35 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 10 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 ++
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 ++
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +++
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 ++
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 ++
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 37 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +++
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 272 ++++-
.../scm/server/OzoneStorageContainerManager.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 136 ++-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 6 +-
.../scm/container/TestContainerManagerImpl.java | 10 +
.../scm/container/TestContainerReportHandler.java | 181 +++-
.../hdds/scm/container/TestReplicationManager.java | 133 ++-
.../container/balancer/TestContainerBalancer.java | 208 ++--
.../algorithms/TestContainerPlacementFactory.java | 8 +
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +++---
.../replication/TestReplicationManagerMetrics.java | 1 -
.../hdds/scm/ha/TestReplicationAnnotation.java | 5 +
.../ha/TestStatefulServiceStateManagerImpl.java | 89 ++
.../scm/node/TestCommandQueueReportHandler.java | 116 +++
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 2 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 ++
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 +++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 125 ++-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 61 ++
.../pipeline/TestWritableECContainerProvider.java | 446 ++++++++
.../safemode/TestHealthyPipelineSafeModeRule.java | 11 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 43 +
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 21 +
hadoop-hdds/tools/pom.xml | 7 +
.../scm/cli/ContainerBalancerStartSubcommand.java | 14 +-
.../hdds/scm/cli/ContainerOperationClient.java | 14 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 3 +
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 ++-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 37 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 73 +-
.../scm/cli/container/TestReportSubCommand.java | 24 +
.../datanode/TestContainerBalancerSubCommand.java | 22 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 ++
.../cli/pipeline/TestListPipelinesSubCommand.java | 192 ++++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 48 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 +++++++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 ++
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 706 +++++++++++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 12 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 234 ++++-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 ++-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 +++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1066 ++++++++++++++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +++
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +++
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 45 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 +++
.../ozone/om/ha/OMFailoverProxyProvider.java | 72 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +---
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 ++
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 ++
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 ++++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 286 +++++-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 ++++++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 61 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 ++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 552 ++++++++++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++++
.../src/test/resources/log4j.properties | 0
hadoop-ozone/csi/pom.xml | 16 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 8 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 3 +-
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 7 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 5 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 41 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 2 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 3 +
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../prepare.robot => basic/single_node.robot} | 15 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 ++
.../dist/src/main/smoketest/ec/basic.robot | 64 ++
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 8 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 2 +-
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
...g4j2.properties => s3g-audit-log4j2.properties} | 8 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 +
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 241 +++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 243 +++--
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 63 +-
.../rooted/ITestRootedOzoneContractRootDir.java | 1 -
.../TestContainerStateManagerIntegration.java | 89 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 435 +++++---
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 +
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 7 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 -
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 ++++++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 ++--
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 3 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 +++++-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 ++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../om/ratis/TestOzoneManagerRatisRequest.java | 35 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 156 ++-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 20 +-
hadoop-ozone/ozone-manager/pom.xml | 11 +
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 +++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 63 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 11 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 144 ++-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 273 +++--
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 ++
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 72 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 ++
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 ++
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 48 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 56 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 77 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 314 ++++++
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../ozone/om/request/OMClientRequestUtils.java | 50 +
.../ozone/om/request/OMKeyRequestFactory.java | 139 ---
.../om/request/bucket/OMBucketCreateRequest.java | 34 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 109 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 74 +-
.../request/file/OMFileCreateRequestWithFSO.java | 15 +-
.../ozone/om/request/file/OMFileRequest.java | 56 +-
.../om/request/key/OMAllocateBlockRequest.java | 45 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 40 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 70 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 15 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 15 -
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 19 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 52 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 137 ++-
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 6 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++
.../S3InitiateMultipartUploadRequest.java | 54 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 43 +-
.../S3MultipartUploadCommitPartRequest.java | 39 +-
.../S3MultipartUploadCompleteRequest.java | 62 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 3 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 16 -
.../om/response/key/OMKeysDeleteResponse.java | 8 +
.../response/key/OMKeysDeleteResponseWithFSO.java | 94 ++
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 5 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 11 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 ++++-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 12 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 12 +
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 ++
.../hadoop/ozone/om/TestKeyDeletingService.java | 142 ++-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 ++
.../apache/hadoop/ozone/om/TestTrashService.java | 2 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../request/TestBucketLayoutAwareOMKeyFactory.java | 166 +++
.../request/file/TestOMDirectoryCreateRequest.java | 37 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../file/TestOMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 88 +-
.../om/request/key/TestOMKeyRenameRequest.java | 14 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 35 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++
.../om/request/key/TestOMKeysRenameRequest.java | 4 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 4 +-
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadAbortRequest.java | 4 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 5 +
.../TestS3MultipartUploadCommitPartRequest.java | 6 +-
.../TestS3MultipartUploadCompleteRequest.java | 10 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +-
.../upgrade/TestOMCancelPrepareRequest.java | 2 +-
.../request/validation/TestRequestValidations.java | 29 +-
.../ozone/om/response/TestCleanupTableInfo.java | 9 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 80 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 53 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +++
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +++-
.../org.apache.hadoop.security.token.DtFetcher | 0
...rg.apache.hadoop.security.token.TokenIdentifier | 0
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 10 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 14 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 2 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 252 +++--
hadoop-ozone/s3gateway/pom.xml | 32 +-
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 ++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 9 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++++
.../hadoop/ozone/s3/OzoneClientProducer.java | 44 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 207 ++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 121 ++-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 232 +++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 46 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 448 ++++++++
.../hadoop/ozone/s3/metrics}/package-info.java | 7 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 75 ++
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../s3gateway/src/main/resources/browser.html | 617 -----------
.../hadoop/ozone/client/OzoneBucketStub.java | 69 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 ++
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 173 ++++
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 9 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 595 +++++++++++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 +++
.../src/test/resources/auditlog.properties | 4 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++
.../ozone/freon/OmBucketReadWriteFileOps.java | 278 +++++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 109 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++
.../ozone/shell/ShellReplicationOptions.java | 20 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 18 +-
628 files changed, 37262 insertions(+), 4953 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ECXceiverClientGrpc.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java => client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockExtendedInputStream.java} (68%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ExtendedInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java} (58%)
copy {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java (100%)
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/ECStreamTestUtil.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStreamProxy.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockLocationInfo.java (54%)
rename hadoop-hdds/{framework => common}/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java (100%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/{AuditLoggerType.java => S3GAction.java} (60%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestECReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerInfo.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/common/helpers/TestExcludeList.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/protocol/commands/TestReconstructionECContainersCommands.java
create mode 100644 hadoop-hdds/docs/content/feature/EC-Chunk-Layout.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reads-With-No-Failures.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reconstructional-Read.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Write-Block-Allocation-in-Containers.png
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.md
copy {hadoop-ozone/client => hadoop-hdds/erasurecode}/pom.xml (57%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/HadoopNativeECAccessorUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder}/package-info.java (73%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/BufferAllocator.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/CodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/ECChunk.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode}/package-info.java (69%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/CoderUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/EncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ErasureCodeNative.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/package-info.java} (50%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/CodecUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GF256.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GaloisField.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/RSUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util}/package-info.java (74%)
create mode 100644 hadoop-hdds/erasurecode/src/main/resources/META-INF/services/org.apache.ozone.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCoderBase.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/package-info.java} (77%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderBenchmark.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestCodecRawCoderMapping.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestDummyRawCoder.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeRSRawCoder.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java} (57%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoder.java} (69%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoderBase.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawCoderBase.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java (51%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoder.java} (75%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder}/package-info.java (83%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/IllegalContainerBalancerStateException.java (52%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/InvalidContainerBalancerConfigurationException.java (50%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{ReplicationActivityStatusMXBean.java => InflightAction.java} (58%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ReplicationManager.java => replication/LegacyReplicationManager.java} (88%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatus.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ByteStringCodec.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
copy hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/{TestSCMContainerPlacementRackAware.java => TestSCMContainerPlacementRackScatter.java} (70%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestStatefulServiceStateManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateMap.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/pipeline/TestListPipelinesSubCommand.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntryPool.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/BlockStreamAccessor.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestKeyInputStreamEC.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/LockUsageInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockMetrics.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
copy hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java => common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransportFactory.java} (58%)
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestQuotaUtil.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
copy hadoop-ozone/{integration-test => common}/src/test/resources/log4j.properties (100%)
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
create mode 100644 hadoop-ozone/dist/src/main/org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory
copy hadoop-ozone/dist/src/main/smoketest/{upgrade/prepare.robot => basic/single_node.robot} (65%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot
copy hadoop-ozone/dist/src/shell/conf/{scm-audit-log4j2.properties => s3g-audit-log4j2.properties} (94%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedDDSWithFSO.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OBSKeyPathLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/RegularBucketLockStrategy.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/BucketLayoutAwareOMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequestUtils.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestBucketLayoutAwareOMKeyFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequestWithFSO.java
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (92%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics}/package-info.java (89%)
copy hadoop-ozone/{dev-support/checks/build.sh => s3gateway/src/main/resources/META-INF/services/org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory} (76%)
mode change 100755 => 100644
delete mode 100644 hadoop-ozone/s3gateway/src/main/resources/browser.html
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
copy {hadoop-hdds/common => hadoop-ozone/s3gateway}/src/test/resources/auditlog.properties (97%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteFileOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java (62%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 17/35: HDDS-5743. [Ozone-Streaming] Add option to write files via streaming api in ofs and o3fs. (#2770)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e1d9db9899310d3a9541239c72babf3e83d45d8f
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Fri Nov 19 12:14:05 2021 +0530
HDDS-5743. [Ozone-Streaming] Add option to write files via streaming api in ofs and o3fs. (#2770)
---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 ++
.../common/src/main/resources/ozone-default.xml | 7 ++
.../apache/hadoop/ozone/client/OzoneBucket.java | 8 ++
.../ozone/client/protocol/ClientProtocol.java | 5 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 20 ++++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 33 +++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 ++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 39 ++++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 8 ++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 +++++++++++++++++++++
11 files changed, 241 insertions(+)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index ee0556dd43..305be444af 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -97,6 +97,13 @@ public final class OzoneConfigKeys {
public static final int DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT
= 9855;
+ /**
+ * Flag to enable ratis streaming on filesystem writes.
+ */
+ public static final String OZONE_FS_DATASTREAM_ENABLE =
+ "ozone.fs.datastream.enable";
+ public static final boolean OZONE_FS_DATASTREAM_ENABLE_DEFAULT = false;
+
/**
* When set to true, allocate a random free port for ozone container, so that
* a mini cluster is able to launch multiple containers on a node.
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index e0732fc4c6..49a81cb968 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3188,4 +3188,11 @@
log level is debug. Ex: "CREATE_CONTAINER,READ_CONTAINER,UPDATE_CONTAINER".
</description>
</property>
+ <property>
+ <name>ozone.fs.datastream.enable</name>
+ <value>false</value>
+ <tag>OZONE, DATANODE</tag>
+ <description> To enable/disable filesystem write via ratis streaming.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index bf7f618b96..3cd0ca7f62 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -952,6 +952,14 @@ public class OzoneBucket extends WithMetadata {
overWrite, recursive);
}
+ public OzoneDataStreamOutput createStreamFile(String keyName, long size,
+ ReplicationConfig replicationConfig, boolean overWrite,
+ boolean recursive) throws IOException {
+ return proxy
+ .createStreamFile(volumeName, name, keyName, size, replicationConfig,
+ overWrite, recursive);
+ }
+
/**
* List the status for a file or a directory and its contents.
*
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 5083f5a78a..6e0cc6602f 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -702,6 +702,11 @@ public interface ClientProtocol {
String keyName, long size, ReplicationConfig replicationConfig,
boolean overWrite, boolean recursive) throws IOException;
+ @SuppressWarnings("checkstyle:parameternumber")
+ OzoneDataStreamOutput createStreamFile(String volumeName, String bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ boolean overWrite, boolean recursive) throws IOException;
+
/**
* List the status for a file or a directory and its contents.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index c3002a5abc..42891c492b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1630,6 +1630,26 @@ public class RpcClient implements ClientProtocol {
return createOutputStream(keySession, UUID.randomUUID().toString());
}
+ @Override
+ public OzoneDataStreamOutput createStreamFile(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, boolean overWrite, boolean recursive)
+ throws IOException {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(size)
+ .setReplicationConfig(replicationConfig)
+ .setAcls(getAclList())
+ .setLatestVersionLocation(getLatestVersionLocation)
+ .build();
+ OpenKeySession keySession =
+ ozoneManagerClient.createFile(keyArgs, overWrite, recursive);
+ return createDataStreamOutput(keySession, UUID.randomUUID().toString(),
+ replicationConfig);
+ }
+
@Override
public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
String keyName, boolean recursive, String startKey, long numEntries)
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index bec00e9299..be919b76fe 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -265,6 +266,38 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter {
return this.bucketReplicationConfig;
}
+ @Override
+ public OzoneFSDataStreamOutput createStreamFile(String key, short replication,
+ boolean overWrite, boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ try {
+ OzoneDataStreamOutput ozoneDataStreamOutput = null;
+ if (replication == ReplicationFactor.ONE.getValue()
+ || replication == ReplicationFactor.THREE.getValue()) {
+
+ ReplicationConfig customReplicationConfig =
+ ReplicationConfig.adjustReplication(bucketReplicationConfig,
+ replication, config);
+ ozoneDataStreamOutput = bucket
+ .createStreamFile(key, 0, customReplicationConfig, overWrite,
+ recursive);
+ } else {
+ ozoneDataStreamOutput = bucket.createStreamFile(
+ key, 0, bucketReplicationConfig, overWrite, recursive);
+ }
+ return new OzoneFSDataStreamOutput(
+ ozoneDataStreamOutput.getByteBufStreamOutput());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
@Override
public void renameKey(String key, String newKeyName) throws IOException {
incrementCounter(Statistic.OBJECTS_RENAMED, 1);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 34961050ad..e669f1c5cc 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -259,6 +260,13 @@ public class BasicOzoneFileSystem extends FileSystem {
private FSDataOutputStream createOutputStream(String key, short replication,
boolean overwrite, boolean recursive) throws IOException {
+ boolean isRatisStreamingEnabled = getConf().getBoolean(
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
+ if (isRatisStreamingEnabled){
+ return new FSDataOutputStream(adapter.createStreamFile(key,
+ replication, overwrite, recursive), statistics);
+ }
return new FSDataOutputStream(adapter.createFile(key,
replication, overwrite, recursive), statistics);
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 052c6c8ba6..5295aa33d6 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -352,6 +353,44 @@ public class BasicRootedOzoneClientAdapterImpl
}
}
+ @Override
+ public OzoneFSDataStreamOutput createStreamFile(String pathStr,
+ short replication, boolean overWrite, boolean recursive)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ if (ofsPath.isRoot() || ofsPath.isVolume() || ofsPath.isBucket()) {
+ throw new IOException("Cannot create file under root or volume.");
+ }
+ String key = ofsPath.getKeyName();
+ try {
+ // Hadoop CopyCommands class always sets recursive to true
+ OzoneBucket bucket = getBucket(ofsPath, recursive);
+ OzoneDataStreamOutput ozoneDataStreamOutput = null;
+ if (replication == ReplicationFactor.ONE.getValue()
+ || replication == ReplicationFactor.THREE.getValue()) {
+
+ ozoneDataStreamOutput = bucket.createStreamFile(key, 0,
+ ReplicationConfig.adjustReplication(
+ clientConfiguredReplicationConfig, replication, config),
+ overWrite, recursive);
+ } else {
+ ozoneDataStreamOutput = bucket.createStreamFile(
+ key, 0, clientConfiguredReplicationConfig, overWrite, recursive);
+ }
+ return new OzoneFSDataStreamOutput(
+ ozoneDataStreamOutput.getByteBufStreamOutput());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
@Override
public void renameKey(String key, String newKeyName) throws IOException {
throw new IOException("OFS doesn't support renameKey, use rename instead.");
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 0360e345b2..0bed09ba8a 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.ozone.OFSPath;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -235,6 +236,13 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
private FSDataOutputStream createOutputStream(String key, short replication,
boolean overwrite, boolean recursive) throws IOException {
+ boolean isRatisStreamingEnabled = getConf().getBoolean(
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
+ if (isRatisStreamingEnabled){
+ return new FSDataOutputStream(adapter.createStreamFile(key,
+ replication, overwrite, recursive), statistics);
+ }
return new FSDataOutputStream(adapter.createFile(key,
replication, overwrite, recursive), statistics);
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
index 31bf351f01..24566cb83f 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
@@ -45,6 +45,9 @@ public interface OzoneClientAdapter {
OzoneFSOutputStream createFile(String key, short replication,
boolean overWrite, boolean recursive) throws IOException;
+ OzoneFSDataStreamOutput createStreamFile(String key, short replication,
+ boolean overWrite, boolean recursive) throws IOException;
+
void renameKey(String key, String newKeyName) throws IOException;
// Users should use rename instead of renameKey in OFS.
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java
new file mode 100644
index 0000000000..515dbca92b
--- /dev/null
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * The ByteBuffer output stream for Ozone file system.
+ */
+public class OzoneFSDataStreamOutput extends OutputStream
+ implements ByteBufferStreamOutput {
+
+ private final ByteBufferStreamOutput byteBufferStreamOutput;
+
+ public OzoneFSDataStreamOutput(
+ ByteBufferStreamOutput byteBufferStreamOutput) {
+ this.byteBufferStreamOutput = byteBufferStreamOutput;
+ }
+
+ /**
+ * Try to write the [off:off + len) slice in ByteBuf b to DataStream.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @throws IOException if an I/O error occurs.
+ */
+ @Override
+ public void write(ByteBuffer b, int off, int len)
+ throws IOException {
+ byteBufferStreamOutput.write(b, off, len);
+ }
+
+ /**
+ * Writes the specified byte to this output stream. The general
+ * contract for <code>write</code> is that one byte is written
+ * to the output stream. The byte to be written is the eight
+ * low-order bits of the argument <code>b</code>. The 24
+ * high-order bits of <code>b</code> are ignored.
+ * <p>
+ * Subclasses of <code>OutputStream</code> must provide an
+ * implementation for this method.
+ *
+ * @param b the <code>byte</code>.
+ * @throws IOException if an I/O error occurs. In particular,
+ * an <code>IOException</code> may be thrown if the
+ * output stream has been closed.
+ */
+ @Override
+ public void write(int b) throws IOException {
+ byte[] singleBytes = new byte[1];
+ singleBytes[0] = (byte) b;
+ byteBufferStreamOutput.write(ByteBuffer.wrap(singleBytes));
+ }
+
+ /**
+ * Flushes this DataStream output and forces any buffered output bytes
+ * to be written out.
+ *
+ * @throws IOException if an I/O error occurs.
+ */
+ @Override
+ public void flush() throws IOException {
+ byteBufferStreamOutput.flush();
+ }
+
+ /**
+ * Closes this stream and releases any system resources associated
+ * with it. If the stream is already closed then invoking this
+ * method has no effect.
+ *
+ * <p> As noted in {@link AutoCloseable#close()}, cases where the
+ * close may fail require careful attention. It is strongly advised
+ * to relinquish the underlying resources and to internally
+ * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+ * the {@code IOException}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ @Override
+ public void close() throws IOException {
+ byteBufferStreamOutput.close();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/35: HDDS-5366. [Ozone-Streaming] Implement stream method to ContainerStateMachine. (#2358). Contributed by mingchao zhao
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 553346e7d22137cd3646c930a4d7984158122af3
Author: micah zhao <mi...@tencent.com>
AuthorDate: Wed Jun 23 23:20:27 2021 +0800
HDDS-5366. [Ozone-Streaming] Implement stream method to ContainerStateMachine. (#2358). Contributed by mingchao zhao
---
.../server/ratis/ContainerStateMachine.java | 25 ++++++++++
.../common/transport/server/ratis/LocalStream.java | 50 +++++++++++++++++++
.../transport/server/ratis/StreamDataChannel.java | 57 ++++++++++++++++++++++
3 files changed, 132 insertions(+)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 494cfe1b13..11095dcd7f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -25,10 +25,12 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collection;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -510,6 +512,29 @@ public class ContainerStateMachine extends BaseStateMachine {
return raftFuture;
}
+ @Override
+ public CompletableFuture<DataStream> stream(RaftClientRequest request) {
+ return CompletableFuture.supplyAsync(() -> {
+ try {
+ ContainerCommandRequestProto requestProto =
+ getContainerCommandRequestProto(gid,
+ request.getMessage().getContent());
+ DispatcherContext context =
+ new DispatcherContext.Builder()
+ .setStage(DispatcherContext.WriteChunkStage.WRITE_DATA)
+ .setContainer2BCSIDMap(container2BCSIDMap)
+ .build();
+
+ ContainerCommandResponseProto response = runCommand(
+ requestProto, context);
+ String path = response.getMessage();
+ return new LocalStream(new StreamDataChannel(Paths.get(path)));
+ } catch (IOException e) {
+ throw new CompletionException("Failed to create data stream", e);
+ }
+ }, executor);
+ }
+
private ExecutorService getChunkExecutor(WriteChunkRequestProto req) {
int i = (int)(req.getBlockID().getLocalID() % chunkExecutors.size());
return chunkExecutors.get(i);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
new file mode 100644
index 0000000000..baae013966
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.transport.server.ratis;
+
+import org.apache.ratis.statemachine.StateMachine;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+
+class LocalStream implements StateMachine.DataStream {
+ private final StateMachine.DataChannel dataChannel;
+
+ LocalStream(StateMachine.DataChannel dataChannel) {
+ this.dataChannel = dataChannel;
+ }
+
+ @Override
+ public StateMachine.DataChannel getDataChannel() {
+ return dataChannel;
+ }
+
+ @Override
+ public CompletableFuture<?> cleanUp() {
+ return CompletableFuture.supplyAsync(() -> {
+ try {
+ dataChannel.close();
+ return true;
+ } catch (IOException e) {
+ throw new CompletionException("Failed to close data channel", e);
+ }
+ });
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java
new file mode 100644
index 0000000000..3df66e26dc
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.transport.server.ratis;
+
+import org.apache.ratis.statemachine.StateMachine;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+
+class StreamDataChannel implements StateMachine.DataChannel {
+ private final Path path;
+ private final RandomAccessFile randomAccessFile;
+
+ StreamDataChannel(Path path) throws FileNotFoundException {
+ this.path = path;
+ this.randomAccessFile = new RandomAccessFile(path.toFile(), "rw");
+ }
+
+ @Override
+ public void force(boolean metadata) throws IOException {
+ randomAccessFile.getChannel().force(metadata);
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ return randomAccessFile.getChannel().write(src);
+ }
+
+ @Override
+ public boolean isOpen() {
+ return randomAccessFile.getChannel().isOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ randomAccessFile.close();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 03/35: HDDS-5481. Fix stream() and link() method in ContainerStateMachine. (#2451)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit d32ae3531802e5b55f95bb1a3e93d080703d2494
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Thu Jul 22 19:46:31 2021 +0800
HDDS-5481. Fix stream() and link() method in ContainerStateMachine. (#2451)
---
.../common/transport/server/ratis/ContainerStateMachine.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index faf2ef7c19..aed81ab5d8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -518,8 +518,7 @@ public class ContainerStateMachine extends BaseStateMachine {
return CompletableFuture.supplyAsync(() -> {
try {
ContainerCommandRequestProto requestProto =
- getContainerCommandRequestProto(gid,
- request.getMessage().getContent());
+ message2ContainerCommandRequestProto(request.getMessage());
DispatcherContext context =
new DispatcherContext.Builder()
.setStage(DispatcherContext.WriteChunkStage.WRITE_DATA)
@@ -536,6 +535,7 @@ public class ContainerStateMachine extends BaseStateMachine {
}, executor);
}
+ @Override
public CompletableFuture<?> link(DataStream stream, LogEntryProto entry) {
return CompletableFuture.supplyAsync(() -> {
if (stream == null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 07/35: HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to ByteBufferStreamOutput (#2603)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3479f54b695f7d8ebeae4dd368ecc506d5f34718
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Wed Sep 8 09:25:17 2021 +0800
HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to ByteBufferStreamOutput (#2603)
---
hadoop-hdds/client/pom.xml | 4 --
.../hdds/scm/storage/BlockDataStreamOutput.java | 26 +++++----
...reamOutput.java => ByteBufferStreamOutput.java} | 15 +++--
.../client/io/BlockDataStreamOutputEntry.java | 51 ++++++++---------
.../ozone/client/io/KeyDataStreamOutput.java | 15 +++--
.../ozone/client/io/OzoneDataStreamOutput.java | 30 +++++-----
.../client/rpc/TestBlockDataStreamOutput.java | 66 +++++++++-------------
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 8 +--
8 files changed, 98 insertions(+), 117 deletions(-)
diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
index 17120fa250..9f2116c96f 100644
--- a/hadoop-hdds/client/pom.xml
+++ b/hadoop-hdds/client/pom.xml
@@ -70,10 +70,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>${spotbugs.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-buffer</artifactId>
- </dependency>
</dependencies>
<build>
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 39ec2f9219..d0419fa0c3 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.storage;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import io.netty.buffer.ByteBuf;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -46,6 +45,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@@ -59,7 +59,7 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putBlockAsync;
/**
- * An {@link ByteBufStreamOutput} used by the REST service in combination
+ * An {@link ByteBufferStreamOutput} used by the REST service in combination
* with the SCMClient to write the value of a key to a sequence
* of container chunks. Writes are buffered locally and periodically written to
* the container as a new chunk. In order to preserve the semantics that
@@ -74,7 +74,7 @@ import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putBlock
* This class encapsulates all state management for buffering and writing
* through to the container.
*/
-public class BlockDataStreamOutput implements ByteBufStreamOutput {
+public class BlockDataStreamOutput implements ByteBufferStreamOutput {
public static final Logger LOG =
LoggerFactory.getLogger(BlockDataStreamOutput.class);
public static final String EXCEPTION_MSG =
@@ -209,16 +209,16 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
}
@Override
- public void write(ByteBuf buf) throws IOException {
+ public void write(ByteBuffer b, int off, int len) throws IOException {
checkOpen();
- if (buf == null) {
+ if (b == null) {
throw new NullPointerException();
}
- final int len = buf.readableBytes();
if (len == 0) {
return;
}
- writeChunkToContainer(buf);
+ writeChunkToContainer(
+ (ByteBuffer) b.asReadOnlyBuffer().position(off).limit(off + len));
writtenDataLength += len;
}
@@ -476,15 +476,17 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
* Writes buffered data as a new chunk to the container and saves chunk
* information to be used later in putKey call.
*
+ * @param buf chunk data to write, from position to limit
* @throws IOException if there is an I/O error while performing the call
* @throws OzoneChecksumException if there is an error while computing
* checksum
*/
- private void writeChunkToContainer(ByteBuf buf)
+ private void writeChunkToContainer(ByteBuffer buf)
throws IOException {
- ChecksumData checksumData = checksum.computeChecksum(buf.nioBuffer());
- int effectiveChunkSize = buf.readableBytes();
+ final int effectiveChunkSize = buf.remaining();
final long offset = chunkOffset.getAndAdd(effectiveChunkSize);
+ ChecksumData checksumData =
+ checksum.computeChecksum(buf.asReadOnlyBuffer());
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(blockID.get().getLocalID() + "_chunk_" + ++chunkIndex)
.setOffset(offset)
@@ -499,8 +501,8 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
CompletableFuture<DataStreamReply> future =
(needSync(offset + effectiveChunkSize) ?
- out.writeAsync(buf.nioBuffer(), StandardWriteOption.SYNC) :
- out.writeAsync(buf.nioBuffer()))
+ out.writeAsync(buf, StandardWriteOption.SYNC) :
+ out.writeAsync(buf))
.whenCompleteAsync((r, e) -> {
if (e != null || !r.isSuccess()) {
if (e == null) {
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
similarity index 82%
rename from hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java
rename to hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
index 7f40737b70..0650a685b6 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
@@ -18,23 +18,24 @@
package org.apache.hadoop.hdds.scm.storage;
-import io.netty.buffer.ByteBuf;
-
import java.io.Closeable;
import java.io.IOException;
+import java.nio.ByteBuffer;
/**
* This interface is for writing an output stream of ByteBuffers.
-* An ByteBufStreamOutput accepts Netty ByteBuf and sends them to some sink.
+* An ByteBufferStreamOutput accepts nio ByteBuffer and sends them to some sink.
*/
-public interface ByteBufStreamOutput extends Closeable {
+public interface ByteBufferStreamOutput extends Closeable {
/**
* Try to write all the bytes in ByteBuf b to DataStream.
*
* @param b the data.
* @exception IOException if an I/O error occurs.
*/
- void write(ByteBuf b) throws IOException;
+ default void write(ByteBuffer b) throws IOException {
+ write(b, b.position(), b.remaining());
+ }
/**
* Try to write the [off:off + len) slice in ByteBuf b to DataStream.
@@ -44,9 +45,7 @@ public interface ByteBufStreamOutput extends Closeable {
* @param len the number of bytes to write.
* @exception IOException if an I/O error occurs.
*/
- default void write(ByteBuf b, int off, int len) throws IOException {
- write(b.slice(off, len));
- }
+ void write(ByteBuffer b, int off, int len) throws IOException;
/**
* Flushes this DataStream output and forces any buffered output bytes
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
index 98907bf8af..f0c3a43e89 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
@@ -18,18 +18,18 @@
package org.apache.hadoop.ozone.client.io;
import com.google.common.annotations.VisibleForTesting;
-import io.netty.buffer.ByteBuf;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
-import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.security.token.Token;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
@@ -37,10 +37,10 @@ import java.util.Collections;
* Helper class used inside {@link BlockDataStreamOutput}.
* */
public final class BlockDataStreamOutputEntry
- implements ByteBufStreamOutput {
+ implements ByteBufferStreamOutput {
private final OzoneClientConfig config;
- private ByteBufStreamOutput byteBufStreamOutput;
+ private ByteBufferStreamOutput byteBufferStreamOutput;
private BlockID blockID;
private final String key;
private final XceiverClientFactory xceiverClientManager;
@@ -61,7 +61,7 @@ public final class BlockDataStreamOutputEntry
OzoneClientConfig config
) {
this.config = config;
- this.byteBufStreamOutput = null;
+ this.byteBufferStreamOutput = null;
this.blockID = blockID;
this.key = key;
this.xceiverClientManager = xceiverClientManager;
@@ -90,63 +90,62 @@ public final class BlockDataStreamOutputEntry
* @throws IOException if xceiverClient initialization fails
*/
private void checkStream() throws IOException {
- if (this.byteBufStreamOutput == null) {
- this.byteBufStreamOutput =
+ if (this.byteBufferStreamOutput == null) {
+ this.byteBufferStreamOutput =
new BlockDataStreamOutput(blockID, xceiverClientManager,
pipeline, config, token);
}
}
@Override
- public void write(ByteBuf b) throws IOException {
+ public void write(ByteBuffer b, int off, int len) throws IOException {
checkStream();
- final int len = b.readableBytes();
- byteBufStreamOutput.write(b);
+ byteBufferStreamOutput.write(b, off, len);
this.currentPosition += len;
}
@Override
public void flush() throws IOException {
- if (this.byteBufStreamOutput != null) {
- this.byteBufStreamOutput.flush();
+ if (this.byteBufferStreamOutput != null) {
+ this.byteBufferStreamOutput.flush();
}
}
@Override
public void close() throws IOException {
- if (this.byteBufStreamOutput != null) {
- this.byteBufStreamOutput.close();
+ if (this.byteBufferStreamOutput != null) {
+ this.byteBufferStreamOutput.close();
// after closing the chunkOutPutStream, blockId would have been
// reconstructed with updated bcsId
this.blockID =
- ((BlockDataStreamOutput) byteBufStreamOutput).getBlockID();
+ ((BlockDataStreamOutput) byteBufferStreamOutput).getBlockID();
}
}
boolean isClosed() {
- if (byteBufStreamOutput != null) {
- return ((BlockDataStreamOutput) byteBufStreamOutput).isClosed();
+ if (byteBufferStreamOutput != null) {
+ return ((BlockDataStreamOutput) byteBufferStreamOutput).isClosed();
}
return false;
}
Collection<DatanodeDetails> getFailedServers() {
- if (byteBufStreamOutput != null) {
+ if (byteBufferStreamOutput != null) {
BlockDataStreamOutput out =
- (BlockDataStreamOutput) this.byteBufStreamOutput;
+ (BlockDataStreamOutput) this.byteBufferStreamOutput;
return out.getFailedServers();
}
return Collections.emptyList();
}
long getWrittenDataLength() {
- if (byteBufStreamOutput != null) {
+ if (byteBufferStreamOutput != null) {
BlockDataStreamOutput out =
- (BlockDataStreamOutput) this.byteBufStreamOutput;
+ (BlockDataStreamOutput) this.byteBufferStreamOutput;
return out.getWrittenDataLength();
} else {
// For a pre allocated block for which no write has been initiated,
- // the ByteBufStreamOutput will be null here.
+ // the ByteBufferStreamOutput will be null here.
// In such cases, the default blockCommitSequenceId will be 0
return 0;
}
@@ -155,7 +154,7 @@ public final class BlockDataStreamOutputEntry
void cleanup(boolean invalidateClient) throws IOException {
checkStream();
BlockDataStreamOutput out =
- (BlockDataStreamOutput) this.byteBufStreamOutput;
+ (BlockDataStreamOutput) this.byteBufferStreamOutput;
out.cleanup(invalidateClient);
}
@@ -163,7 +162,7 @@ public final class BlockDataStreamOutputEntry
void writeOnRetry(long len) throws IOException {
checkStream();
BlockDataStreamOutput out =
- (BlockDataStreamOutput) this.byteBufStreamOutput;
+ (BlockDataStreamOutput) this.byteBufferStreamOutput;
out.writeOnRetry(len);
this.currentPosition += len;
@@ -231,8 +230,8 @@ public final class BlockDataStreamOutputEntry
}
@VisibleForTesting
- public ByteBufStreamOutput getByteBufStreamOutput() {
- return byteBufStreamOutput;
+ public ByteBufferStreamOutput getByteBufStreamOutput() {
+ return byteBufferStreamOutput;
}
public BlockID getBlockID() {
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
index c37f9cd51d..9bba89d0a8 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.client.io;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import io.netty.buffer.ByteBuf;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.ReplicationConfig;
@@ -32,7 +31,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -48,6 +47,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -63,7 +63,7 @@ import java.util.stream.Collectors;
*
* TODO : currently not support multi-thread access.
*/
-public class KeyDataStreamOutput implements ByteBufStreamOutput {
+public class KeyDataStreamOutput implements ByteBufferStreamOutput {
private OzoneClientConfig config;
@@ -185,17 +185,16 @@ public class KeyDataStreamOutput implements ByteBufStreamOutput {
}
@Override
- public void write(ByteBuf b) throws IOException {
+ public void write(ByteBuffer b, int off, int len) throws IOException {
checkNotClosed();
if (b == null) {
throw new NullPointerException();
}
- final int len = b.readableBytes();
- handleWrite(b, b.readerIndex(), len, false);
+ handleWrite(b, off, len, false);
writeOffset += len;
}
- private void handleWrite(ByteBuf b, int off, long len, boolean retry)
+ private void handleWrite(ByteBuffer b, int off, long len, boolean retry)
throws IOException {
while (len > 0) {
try {
@@ -227,7 +226,7 @@ public class KeyDataStreamOutput implements ByteBufStreamOutput {
}
private int writeToDataStreamOutput(BlockDataStreamOutputEntry current,
- boolean retry, long len, ByteBuf b, int writeLen, int off,
+ boolean retry, long len, ByteBuffer b, int writeLen, int off,
long currentPos) throws IOException {
try {
if (retry) {
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
index 378b86872e..d40ac2b332 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
@@ -16,55 +16,55 @@
*/
package org.apache.hadoop.ozone.client.io;
-import io.netty.buffer.ByteBuf;
-import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import java.io.IOException;
+import java.nio.ByteBuffer;
/**
* OzoneDataStreamOutput is used to write data into Ozone.
* It uses SCM's {@link KeyDataStreamOutput} for writing the data.
*/
-public class OzoneDataStreamOutput implements ByteBufStreamOutput {
+public class OzoneDataStreamOutput implements ByteBufferStreamOutput {
- private final ByteBufStreamOutput byteBufStreamOutput;
+ private final ByteBufferStreamOutput byteBufferStreamOutput;
/**
* Constructs OzoneDataStreamOutput with KeyDataStreamOutput.
*
- * @param byteBufStreamOutput
+ * @param byteBufferStreamOutput the underlying ByteBufferStreamOutput
*/
- public OzoneDataStreamOutput(ByteBufStreamOutput byteBufStreamOutput) {
- this.byteBufStreamOutput = byteBufStreamOutput;
+ public OzoneDataStreamOutput(ByteBufferStreamOutput byteBufferStreamOutput) {
+ this.byteBufferStreamOutput = byteBufferStreamOutput;
}
@Override
- public void write(ByteBuf b) throws IOException {
- byteBufStreamOutput.write(b);
+ public void write(ByteBuffer b, int off, int len) throws IOException {
+ byteBufferStreamOutput.write(b, off, len);
}
@Override
public synchronized void flush() throws IOException {
- byteBufStreamOutput.flush();
+ byteBufferStreamOutput.flush();
}
@Override
public synchronized void close() throws IOException {
//commitKey can be done here, if needed.
- byteBufStreamOutput.close();
+ byteBufferStreamOutput.close();
}
public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() {
- if (byteBufStreamOutput instanceof KeyDataStreamOutput) {
+ if (byteBufferStreamOutput instanceof KeyDataStreamOutput) {
return ((KeyDataStreamOutput)
- byteBufStreamOutput).getCommitUploadPartInfo();
+ byteBufferStreamOutput).getCommitUploadPartInfo();
}
// Otherwise return null.
return null;
}
- public ByteBufStreamOutput getByteBufStreamOutput() {
- return byteBufStreamOutput;
+ public ByteBufferStreamOutput getByteBufStreamOutput() {
+ return byteBufferStreamOutput;
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 4d52d89490..6d5401d651 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.client.rpc;
-import io.netty.buffer.Unpooled;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -38,6 +37,7 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -126,48 +126,38 @@ public class TestBlockDataStreamOutput {
}
}
+ @Test
+ public void testHalfChunkWrite() throws Exception {
+ testWrite(chunkSize / 2);
+ }
+
+ @Test
+ public void testSingleChunkWrite() throws Exception {
+ testWrite(chunkSize);
+ }
+
@Test
public void testMultiChunkWrite() throws Exception {
- // write data less than 1 chunk size use streaming.
- String keyName1 = getKeyName();
- OzoneDataStreamOutput key1 = createKey(
- keyName1, ReplicationType.RATIS, 0);
- int dataLength1 = chunkSize/2;
- byte[] data1 =
- ContainerTestHelper.getFixedLengthString(keyString, dataLength1)
- .getBytes(UTF_8);
- key1.write(Unpooled.copiedBuffer(data1));
- // now close the stream, It will update the key length.
- key1.close();
- validateData(keyName1, data1);
-
- // write data more than 1 chunk size use streaming.
- String keyName2 = getKeyName();
- OzoneDataStreamOutput key2 = createKey(
- keyName2, ReplicationType.RATIS, 0);
- int dataLength2 = chunkSize + 50;
- byte[] data2 =
- ContainerTestHelper.getFixedLengthString(keyString, dataLength2)
- .getBytes(UTF_8);
- key2.write(Unpooled.copiedBuffer(data2));
- // now close the stream, It will update the key length.
- key2.close();
- validateData(keyName2, data2);
-
- // write data more than 1 block size use streaming.
- String keyName3 = getKeyName();
- OzoneDataStreamOutput key3 = createKey(
- keyName3, ReplicationType.RATIS, 0);
- int dataLength3 = blockSize + 50;
- byte[] data3 =
- ContainerTestHelper.getFixedLengthString(keyString, dataLength3)
+ testWrite(chunkSize + 50);
+ }
+
+ @Test
+ public void testMultiBlockWrite() throws Exception {
+ testWrite(blockSize + 50);
+ }
+
+ private void testWrite(int dataLength) throws Exception {
+ String keyName = getKeyName();
+ OzoneDataStreamOutput key = createKey(
+ keyName, ReplicationType.RATIS, 0);
+ byte[] data =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
- key3.write(Unpooled.copiedBuffer(data3));
+ key.write(ByteBuffer.wrap(data));
// now close the stream, It will update the key length.
- key3.close();
- validateData(keyName3, data3);
+ key.close();
+ validateData(keyName, data);
}
-
private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
long size) throws Exception {
return TestHelper.createStreamKey(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index e4b842eaca..c34783dbdd 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -29,8 +29,6 @@ import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.Map;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.io.IOUtils;
@@ -117,10 +115,8 @@ public class PutKeyHandler extends KeyHandler {
long off = 0;
while (len > 0) {
long writeLen = Math.min(len, chunkSize);
- ByteBuffer segment =
- ch.map(FileChannel.MapMode.READ_ONLY, off, writeLen);
- ByteBuf buf = Unpooled.wrappedBuffer(segment);
- out.write(buf);
+ ByteBuffer bb = ch.map(FileChannel.MapMode.READ_ONLY, off, writeLen);
+ out.write(bb);
off += writeLen;
len -= writeLen;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 20/35: HDDS-6130. [Ozone-Streaming] When releaseBuffers will get “Couldn 't find the required future” (#2939)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit d9d81f5b97e74cf86499df343782cddbfd5c29d4
Author: micah zhao <mi...@tencent.com>
AuthorDate: Fri Dec 24 01:18:48 2021 +0800
HDDS-6130. [Ozone-Streaming] When releaseBuffers will get “Couldn 't find the required future” (#2939)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 92 ++++++++++------------
.../hdds/scm/storage/StreamCommitWatcher.java | 28 -------
2 files changed, 42 insertions(+), 78 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 9fb1340527..84968b68d5 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
@@ -115,6 +116,9 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// Also, corresponding to the logIndex, the corresponding list of buffers will
// be released from the buffer pool.
private final StreamCommitWatcher commitWatcher;
+ private final AtomicReference<CompletableFuture<
+ ContainerCommandResponseProto>> putBlockFuture
+ = new AtomicReference<>(CompletableFuture.completedFuture(null));
private final List<DatanodeDetails> failedServers;
private final Checksum checksum;
@@ -381,8 +385,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
* @param force true if no data was written since most recent putBlock and
* stream is being closed
*/
- private CompletableFuture<ContainerProtos.
- ContainerCommandResponseProto> executePutBlock(boolean close,
+ private void executePutBlock(boolean close,
boolean force) throws IOException {
checkOpen();
long flushPos = totalDataFlushedLength;
@@ -399,56 +402,54 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
dataStreamCloseReply = out.closeAsync();
}
- CompletableFuture<ContainerProtos.
- ContainerCommandResponseProto> flushFuture = null;
try {
BlockData blockData = containerBlockData.build();
XceiverClientReply asyncReply =
putBlockAsync(xceiverClient, blockData, close, token);
- CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
- asyncReply.getResponse();
- flushFuture = future.thenApplyAsync(e -> {
- try {
- validateResponse(e);
- } catch (IOException sce) {
- throw new CompletionException(sce);
- }
- // if the ioException is not set, putBlock is successful
- if (getIoException() == null && !force) {
- BlockID responseBlockID = BlockID.getFromProtobuf(
- e.getPutBlock().getCommittedBlockLength().getBlockID());
- Preconditions.checkState(blockID.get().getContainerBlockID()
- .equals(responseBlockID.getContainerBlockID()));
- // updates the bcsId of the block
- blockID.set(responseBlockID);
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Adding index " + asyncReply.getLogIndex() + " commitMap size "
+ final CompletableFuture<ContainerCommandResponseProto> flushFuture
+ = asyncReply.getResponse().thenApplyAsync(e -> {
+ try {
+ validateResponse(e);
+ } catch (IOException sce) {
+ throw new CompletionException(sce);
+ }
+ // if the ioException is not set, putBlock is successful
+ if (getIoException() == null && !force) {
+ BlockID responseBlockID = BlockID.getFromProtobuf(
+ e.getPutBlock().getCommittedBlockLength().getBlockID());
+ Preconditions.checkState(blockID.get().getContainerBlockID()
+ .equals(responseBlockID.getContainerBlockID()));
+ // updates the bcsId of the block
+ blockID.set(responseBlockID);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding index " + asyncReply.getLogIndex() +
+ " commitMap size "
+ commitWatcher.getCommitInfoMapSize() + " flushLength "
+ flushPos + " blockID " + blockID);
- }
- // for standalone protocol, logIndex will always be 0.
- commitWatcher
- .updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
- }
- return e;
- }, responseExecutor).exceptionally(e -> {
- if (LOG.isDebugEnabled()) {
- LOG.debug("putBlock failed for blockID {} with exception {}",
- blockID, e.getLocalizedMessage());
- }
- CompletionException ce = new CompletionException(e);
- setIoException(ce);
- throw ce;
- });
+ }
+ // for standalone protocol, logIndex will always be 0.
+ commitWatcher
+ .updateCommitInfoMap(asyncReply.getLogIndex(),
+ byteBufferList);
+ }
+ return e;
+ }, responseExecutor).exceptionally(e -> {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("putBlock failed for blockID {} with exception {}",
+ blockID, e.getLocalizedMessage());
+ }
+ CompletionException ce = new CompletionException(e);
+ setIoException(ce);
+ throw ce;
+ });
+ putBlockFuture.updateAndGet(f -> f.thenCombine(flushFuture,
+ (previous, current) -> current));
} catch (IOException | ExecutionException e) {
throw new IOException(EXCEPTION_MSG + e.toString(), e);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
handleInterruptedException(ex, false);
}
- commitWatcher.getFutureMap().put(flushPos, flushFuture);
- return flushFuture;
}
@Override
@@ -484,7 +485,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// data since latest flush - we need to send the "EOF" flag
executePutBlock(true, true);
}
- waitOnFlushFutures();
+ putBlockFuture.get().get();
watchForCommit(false);
// just check again if the exception is hit while waiting for the
// futures to ensure flush has indeed succeeded
@@ -512,15 +513,6 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
}
- private void waitOnFlushFutures()
- throws InterruptedException, ExecutionException {
- CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(
- commitWatcher.getFutureMap().values().toArray(
- new CompletableFuture[commitWatcher.getFutureMap().size()]));
- // wait for all the transactions to complete
- combinedFuture.get();
- }
-
private void validateResponse(
ContainerProtos.ContainerCommandResponseProto responseProto)
throws IOException {
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
index 9ae604e951..1820416d32 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
@@ -25,7 +25,6 @@
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.slf4j.Logger;
@@ -35,9 +34,6 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
@@ -59,18 +55,12 @@ public class StreamCommitWatcher {
// by all servers
private long totalAckDataLength;
- // future Map to hold up all putBlock futures
- private ConcurrentHashMap<Long,
- CompletableFuture<ContainerCommandResponseProto>>
- futureMap;
-
private XceiverClientSpi xceiverClient;
public StreamCommitWatcher(XceiverClientSpi xceiverClient,
List<StreamBuffer> bufferList) {
this.xceiverClient = xceiverClient;
commitIndexMap = new ConcurrentSkipListMap<>();
- futureMap = new ConcurrentHashMap<>();
this.bufferList = bufferList;
totalAckDataLength = 0;
}
@@ -180,15 +170,6 @@ public class StreamCommitWatcher {
final long length =
buffers.stream().mapToLong(StreamBuffer::position).sum();
totalAckDataLength += length;
- // clear the future object from the future Map
- final CompletableFuture<ContainerCommandResponseProto> remove =
- futureMap.remove(totalAckDataLength);
- if (remove == null) {
- LOG.error("Couldn't find required future for " + totalAckDataLength);
- for (Long key : futureMap.keySet()) {
- LOG.error("Existing acknowledged data: " + key);
- }
- }
for (StreamBuffer byteBuffer : buffers) {
bufferList.remove(byteBuffer);
}
@@ -209,19 +190,10 @@ public class StreamCommitWatcher {
return ioException;
}
- public ConcurrentMap<Long,
- CompletableFuture<
- ContainerCommandResponseProto>> getFutureMap() {
- return futureMap;
- }
-
public void cleanup() {
if (commitIndexMap != null) {
commitIndexMap.clear();
}
- if (futureMap != null) {
- futureMap.clear();
- }
commitIndexMap = null;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 31/35: HDDS-6461. Update Ratis version to 2.3.0-da5d868-SNAPSHOT. (#3205)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit ae91a3f869b135e862369a3adb860426ee5629d8
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Thu Mar 17 10:20:55 2022 +0800
HDDS-6461. Update Ratis version to 2.3.0-da5d868-SNAPSHOT. (#3205)
---
.../org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +++++++------
pom.xml | 2 +-
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
index 0cb3e65535..6a0a615a33 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
@@ -34,9 +34,6 @@ import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.tag.Flaky;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.ratis.grpc.client.GrpcClientProtocolService;
import org.apache.ratis.protocol.ClientId;
import org.apache.ratis.protocol.GroupInfoReply;
import org.apache.ratis.protocol.GroupInfoRequest;
@@ -46,7 +43,9 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
/**
* Test pipeline leader information is correctly used.
@@ -55,7 +54,7 @@ import org.slf4j.LoggerFactory;
public class TestRatisPipelineLeader {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
- private static final org.slf4j.Logger LOG =
+ private static final Logger LOG =
LoggerFactory.getLogger(TestRatisPipelineLeader.class);
@BeforeAll
@@ -97,9 +96,11 @@ public class TestRatisPipelineLeader {
// Verify client connects to Leader without NotLeaderException
XceiverClientRatis xceiverClientRatis =
XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf);
- Logger.getLogger(GrpcClientProtocolService.class).setLevel(Level.DEBUG);
+ final Logger log = LoggerFactory.getLogger(
+ "org.apache.ratis.grpc.server.GrpcClientProtocolService");
+ GenericTestUtils.setLogLevel(log, Level.DEBUG);
GenericTestUtils.LogCapturer logCapturer =
- GenericTestUtils.LogCapturer.captureLogs(GrpcClientProtocolService.LOG);
+ GenericTestUtils.LogCapturer.captureLogs(log);
xceiverClientRatis.connect();
ContainerProtocolCalls.createContainer(xceiverClientRatis, 1L, null);
logCapturer.stopCapturing();
diff --git a/pom.xml b/pom.xml
index 3f93e9683c..54b89bb2a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<declared.ozone.version>${ozone.version}</declared.ozone.version>
<!-- Apache Ratis version -->
- <ratis.version>2.3.0-94db58b-SNAPSHOT</ratis.version>
+ <ratis.version>2.3.0-da5d868-SNAPSHOT</ratis.version>
<!-- Apache Ratis thirdparty version -->
<ratis.thirdparty.version>0.7.0</ratis.thirdparty.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 26/35: HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support FlushDelay. (#3002)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 32613fba33fbe0f0e37192d7567d80d34266c80a
Author: micah zhao <mi...@tencent.com>
AuthorDate: Mon Feb 14 20:58:11 2022 +0800
HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support FlushDelay. (#3002)
---
.../apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 10 +++++++++-
.../hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java | 1 -
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index a3fe1c2479..9ac43300f7 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -429,7 +429,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
} else {
byteBufferList = null;
}
- flush();
+ waitFuturesComplete();
if (close) {
dataStreamCloseReply = out.closeAsync();
}
@@ -485,8 +485,16 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
@Override
public void flush() throws IOException {
+ if (xceiverClientFactory != null && xceiverClient != null
+ && !config.isStreamBufferFlushDelay()) {
+ waitFuturesComplete();
+ }
+ }
+
+ public void waitFuturesComplete() throws IOException {
try {
CompletableFuture.allOf(futures.toArray(EMPTY_FUTURE_ARRAY)).get();
+ futures.clear();
} catch (Exception e) {
LOG.warn("Failed to write all chunks through stream: " + e);
throw new IOException(e);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 696ab92ab7..21003374d7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -89,7 +89,6 @@ public class TestBlockDataStreamOutput {
blockSize = 2 * maxFlushSize;
OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
conf.setFromObject(clientConfig);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/35: HDDS-5452. Add link method to ContainerStateMachine for Ratis streaming (#2422)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 133557f3c0b599d1654864aaaf59e9105e62dc21
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Sun Jul 18 12:18:10 2021 +0800
HDDS-5452. Add link method to ContainerStateMachine for Ratis streaming (#2422)
---
.../transport/server/ratis/ContainerStateMachine.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 11095dcd7f..faf2ef7c19 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -92,6 +92,7 @@ import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferExce
import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
import org.apache.ratis.util.TaskQueue;
import org.apache.ratis.util.function.CheckedSupplier;
+import org.apache.ratis.util.JavaUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -535,6 +536,22 @@ public class ContainerStateMachine extends BaseStateMachine {
}, executor);
}
+ public CompletableFuture<?> link(DataStream stream, LogEntryProto entry) {
+ return CompletableFuture.supplyAsync(() -> {
+ if (stream == null) {
+ return JavaUtils.completeExceptionally(
+ new IllegalStateException("DataStream is null"));
+ }
+ if (stream.getDataChannel().isOpen()) {
+ return JavaUtils.completeExceptionally(
+ new IllegalStateException(
+ "DataStream: " + stream + " is not closed properly"));
+ } else {
+ return CompletableFuture.completedFuture(null);
+ }
+ }, executor);
+ }
+
private ExecutorService getChunkExecutor(WriteChunkRequestProto req) {
int i = (int)(req.getBlockID().getLocalID() % chunkExecutors.size());
return chunkExecutors.get(i);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 32/35: HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters. (#3207)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit eeeddff8b93db4d9ac6b623f1c81b2b9bb08c9b7
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Sun Mar 27 17:07:28 2022 +0800
HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters. (#3207)
---
.../common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index d044026322..ca3170985e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -256,6 +256,7 @@ public final class RatisHelper {
GrpcTlsConfig tlsConfig) {
if (tlsConfig != null) {
GrpcConfigKeys.Client.setTlsConf(parameters, tlsConfig);
+ NettyConfigKeys.DataStream.Client.setTlsConf(parameters, tlsConfig);
}
}
@@ -266,6 +267,8 @@ public final class RatisHelper {
GrpcConfigKeys.Server.setTlsConf(parameters, serverConf);
GrpcConfigKeys.TLS.setConf(parameters, serverConf);
setAdminTlsConf(parameters, serverConf);
+
+ NettyConfigKeys.DataStream.Server.setTlsConf(parameters, serverConf);
}
setClientTlsConf(parameters, clientConf);
return parameters;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 24/35: HDDS-6138.[Ozone-Streaming] Define a limit on the size of the retry bufferList. (#2946)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1bbc76f47ab63f1174327e4357841879d1ae4a23
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Thu Feb 10 14:36:03 2022 +0530
HDDS-6138.[Ozone-Streaming] Define a limit on the size of the retry bufferList. (#2946)
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +++++++++---------
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 40 +++++++++++++++++-----
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 12 +++----
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 11 +++---
.../client/rpc/TestBlockDataStreamOutput.java | 14 ++++----
6 files changed, 67 insertions(+), 45 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index 5249e53644..ffc4a3ef5a 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -66,14 +66,6 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private int streamBufferSize = 4 * 1024 * 1024;
- @Config(key = "datastream.max.buffer.size",
- defaultValue = "4MB",
- type = ConfigType.SIZE,
- description = "The maximum size of the ByteBuffer "
- + "(used via ratis streaming)",
- tags = ConfigTag.CLIENT)
- private int dataStreamMaxBufferSize = 4 * 1024 * 1024;
-
@Config(key = "datastream.buffer.flush.size",
defaultValue = "16MB",
type = ConfigType.SIZE,
@@ -89,6 +81,15 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private int dataStreamMinPacketSize = 1024 * 1024;
+ @Config(key = "datastream.window.size",
+ defaultValue = "64MB",
+ type = ConfigType.SIZE,
+ description = "Maximum size of BufferList(used for retry) size per " +
+ "BlockDataStreamOutput instance",
+ tags = ConfigTag.CLIENT)
+ private long streamWindowSize = 64 * 1024 * 1024;
+
+
@Config(key = "stream.buffer.increment",
defaultValue = "0B",
type = ConfigType.SIZE,
@@ -243,14 +244,6 @@ public class OzoneClientConfig {
this.streamBufferSize = streamBufferSize;
}
- public int getDataStreamMaxBufferSize() {
- return dataStreamMaxBufferSize;
- }
-
- public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
- this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
- }
-
public boolean isStreamBufferFlushDelay() {
return streamBufferFlushDelay;
}
@@ -275,6 +268,14 @@ public class OzoneClientConfig {
this.dataStreamMinPacketSize = dataStreamMinPacketSize;
}
+ public long getStreamWindowSize() {
+ return streamWindowSize;
+ }
+
+ public void setStreamWindowSize(long streamWindowSize) {
+ this.streamWindowSize = streamWindowSize;
+ }
+
public int getMaxRetryCount() {
return maxRetryCount;
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index f37cd1c94a..2ad6b69064 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -122,7 +122,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
this.ozoneConfiguration = configuration;
}
- private void updateCommitInfosMap(
+ public void updateCommitInfosMap(
Collection<RaftProtos.CommitInfoProto> commitInfoProtos) {
// if the commitInfo map is empty, just update the commit indexes for each
// of the servers
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index ec925d1e6a..a3fe1c2479 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -52,7 +52,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
@@ -116,9 +118,9 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// Also, corresponding to the logIndex, the corresponding list of buffers will
// be released from the buffer pool.
private final StreamCommitWatcher commitWatcher;
- private final AtomicReference<CompletableFuture<
- ContainerCommandResponseProto>> putBlockFuture
- = new AtomicReference<>(CompletableFuture.completedFuture(null));
+
+ private Queue<CompletableFuture<ContainerCommandResponseProto>>
+ putBlockFutures = new LinkedList<>();
private final List<DatanodeDetails> failedServers;
private final Checksum checksum;
@@ -307,14 +309,33 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
private void doFlushIfNeeded() throws IOException {
- Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
- .getDataStreamMaxBufferSize());
long boundary = config.getDataStreamBufferFlushSize() / config
- .getDataStreamMaxBufferSize();
+ .getDataStreamMinPacketSize();
+ // streamWindow is the maximum number of buffers that
+ // are allowed to exist in the bufferList. If buffers in
+ // the list exceed this limit , client will till it gets
+ // one putBlockResponse (first index) . This is similar to
+ // the bufferFull condition in async write path.
+ long streamWindow = config.getStreamWindowSize() / config
+ .getDataStreamMinPacketSize();
if (!bufferList.isEmpty() && bufferList.size() % boundary == 0) {
updateFlushLength();
executePutBlock(false, false);
}
+ if (bufferList.size()==streamWindow){
+ try {
+ checkOpen();
+ if (!putBlockFutures.isEmpty()) {
+ putBlockFutures.remove().get();
+ }
+ } catch (ExecutionException e) {
+ handleExecutionException(e);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ handleInterruptedException(ex, true);
+ }
+ watchForCommit(true);
+ }
}
private void updateFlushLength() {
@@ -453,8 +474,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
setIoException(ce);
throw ce;
});
- putBlockFuture.updateAndGet(f -> f.thenCombine(flushFuture,
- (previous, current) -> current));
+ putBlockFutures.add(flushFuture);
} catch (IOException | ExecutionException e) {
throw new IOException(EXCEPTION_MSG + e.toString(), e);
} catch (InterruptedException ex) {
@@ -496,7 +516,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// data since latest flush - we need to send the "EOF" flag
executePutBlock(true, true);
}
- putBlockFuture.get().get();
+ CompletableFuture.allOf(putBlockFutures.toArray(EMPTY_FUTURE_ARRAY)).get();
watchForCommit(false);
// just check again if the exception is hit while waiting for the
// futures to ensure flush has indeed succeeded
@@ -638,6 +658,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
CompletionException ce = new CompletionException(msg, e);
setIoException(ce);
throw ce;
+ } else if (r.isSuccess()) {
+ xceiverClient.updateCommitInfosMap(r.getCommitInfos());
}
}, responseExecutor);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 5c0e74f80b..dc7a59b30a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -324,7 +324,7 @@ public interface MiniOzoneCluster {
protected OptionalInt streamBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferFlushSize = Optional.empty();
protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
- protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
+ protected Optional<Long> datastreamWindowSize= Optional.empty();
protected Optional<Long> streamBufferMaxSize = Optional.empty();
protected OptionalInt dataStreamMinPacketSize = OptionalInt.empty();
protected Optional<Long> blockSize = Optional.empty();
@@ -560,11 +560,6 @@ public interface MiniOzoneCluster {
return this;
}
- public Builder setDataStreamBufferMaxSize(int size) {
- dataStreamMaxBufferSize = OptionalInt.of(size);
- return this;
- }
-
public Builder setDataStreamBufferFlushize(long size) {
dataStreamBufferFlushSize = Optional.of(size);
return this;
@@ -575,6 +570,11 @@ public interface MiniOzoneCluster {
return this;
}
+ public Builder setDataStreamStreamWindowSize(long size) {
+ datastreamWindowSize = Optional.of(size);
+ return this;
+ }
+
/**
* Sets the block size for stream buffer.
*
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ce619aac59..66a8eef564 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -664,12 +664,12 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!dataStreamBufferFlushSize.isPresent()) {
dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
}
- if (!dataStreamMaxBufferSize.isPresent()) {
- dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
- }
if (!dataStreamMinPacketSize.isPresent()) {
dataStreamMinPacketSize = OptionalInt.of(chunkSize.get()/4);
}
+ if (!datastreamWindowSize.isPresent()) {
+ datastreamWindowSize = Optional.of((long) 8 * chunkSize.get());
+ }
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
@@ -688,12 +688,11 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
clientConfig.setDataStreamBufferFlushSize(Math.round(
streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
- clientConfig.setDataStreamMaxBufferSize((int) Math.round(
- streamBufferSizeUnit.get()
- .toBytes(dataStreamMaxBufferSize.getAsInt())));
clientConfig.setDataStreamMinPacketSize((int) Math.round(
streamBufferSizeUnit.get()
.toBytes(dataStreamMinPacketSize.getAsInt())));
+ clientConfig.setStreamWindowSize(Math.round(
+ streamBufferSizeUnit.get().toBytes(datastreamWindowSize.get())));
conf.setFromObject(clientConfig);
conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index c6a3c32d2b..696ab92ab7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -106,9 +106,9 @@ public class TestBlockDataStreamOutput {
.setStreamBufferFlushSize(flushSize)
.setStreamBufferMaxSize(maxFlushSize)
.setDataStreamBufferFlushize(maxFlushSize)
- .setDataStreamBufferMaxSize(chunkSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
- .setDataStreamMinPacketSize(2*chunkSize/5)
+ .setDataStreamMinPacketSize(chunkSize)
+ .setDataStreamStreamWindowSize(5*chunkSize)
.build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
@@ -195,7 +195,7 @@ public class TestBlockDataStreamOutput {
@Test
public void testPutBlockAtBoundary() throws Exception {
- int dataLength = 200;
+ int dataLength = 500;
XceiverClientMetrics metrics =
XceiverClientManager.getXceiverClientMetrics();
long putBlockCount = metrics.getContainerOpCountMetrics(
@@ -213,8 +213,8 @@ public class TestBlockDataStreamOutput {
metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
key.close();
- // Since data length is 200 , first putBlock will be at 160(flush boundary)
- // and the other at 200
+ // Since data length is 500 , first putBlock will be at 400(flush boundary)
+ // and the other at 500
Assert.assertTrue(
metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
== putBlockCount + 2);
@@ -242,10 +242,10 @@ public class TestBlockDataStreamOutput {
long writeChunkCount =
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
byte[] data =
- ContainerTestHelper.getFixedLengthString(keyString, chunkSize / 5)
+ ContainerTestHelper.getFixedLengthString(keyString, chunkSize / 2)
.getBytes(UTF_8);
key.write(ByteBuffer.wrap(data));
- // minPacketSize= 40, so first write of 20 wont trigger a writeChunk
+ // minPacketSize= 100, so first write of 50 wont trigger a writeChunk
Assert.assertEquals(writeChunkCount,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
key.write(ByteBuffer.wrap(data));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 35/35: HDDS-5666. Add option to createKey via streaming api in Freon (#2574)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 74ce5363bd890cba0201adcf3101c2915839c814
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Fri Apr 8 17:38:09 2022 +0530
HDDS-5666. Add option to createKey via streaming api in Freon (#2574)
---
.../hadoop/ozone/freon/ContentGenerator.java | 18 ++++++++++++++
.../ozone/freon/OzoneClientKeyGenerator.java | 29 +++++++++++++++++++++-
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
index 542634c488..8adf0cde4e 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
@@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.freon;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
/**
* Utility class to write random keys from a limited buffer.
@@ -80,6 +82,22 @@ public class ContentGenerator {
}
}
+ /**
+ * Write the required bytes to the streaming output stream.
+ */
+ public void write(OzoneDataStreamOutput out) throws IOException {
+ for (long nrRemaining = keySize;
+ nrRemaining > 0; nrRemaining -= bufferSize) {
+ int curSize = (int) Math.min(bufferSize, nrRemaining);
+ for (int i = 0; i < curSize; i += copyBufferSize) {
+ ByteBuffer bb =
+ ByteBuffer.wrap(buffer, i, Math.min(copyBufferSize, curSize - i));
+ out.write(bb);
+ }
+ }
+ out.close();
+ }
+
@VisibleForTesting
byte[] getBuffer() {
return buffer;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
index 74cd0d0b37..6ab5c03009 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
@@ -24,10 +24,12 @@ import java.util.concurrent.Callable;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import com.codahale.metrics.Timer;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
@@ -74,6 +76,12 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator
@Mixin
private FreonReplicationOptions replication;
+ @Option(
+ names = {"--enable-streaming", "--stream"},
+ description = "Specify whether the write will be through ratis streaming"
+ )
+ private boolean enableRatisStreaming = false;
+
private Timer timer;
private OzoneBucket bucket;
@@ -101,7 +109,11 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator
timer = getMetrics().timer("key-create");
- runTests(this::createKey);
+ if (enableRatisStreaming) {
+ runTests(this::createStreamKey);
+ } else {
+ runTests(this::createKey);
+ }
}
return null;
}
@@ -118,4 +130,19 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator
return null;
});
}
+
+ private void createStreamKey(long counter) throws Exception {
+ final ReplicationConfig replicationConfig = ReplicationConfig
+ .fromProtoTypeAndFactor(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+ final String key = generateObjectName(counter);
+
+ timer.time(() -> {
+ try (OzoneDataStreamOutput stream = bucket
+ .createStreamKey(key, keySize, replicationConfig, metadata)) {
+ contentGenerator.write(stream);
+ }
+ return null;
+ });
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 23/35: HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT version (#3059)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a65a63ee7d67f89e046eae50c01d434cb6ce0735
Author: hao guo <gu...@360.cn>
AuthorDate: Thu Feb 10 09:04:29 2022 +0800
HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT version (#3059)
---
hadoop-ozone/dist/src/main/license/update-jar-report.sh | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/license/update-jar-report.sh b/hadoop-ozone/dist/src/main/license/update-jar-report.sh
index dfd68a264d..6cc6238f40 100755
--- a/hadoop-ozone/dist/src/main/license/update-jar-report.sh
+++ b/hadoop-ozone/dist/src/main/license/update-jar-report.sh
@@ -30,4 +30,4 @@ cd "$OZONE_DIST_DIR"
#sed expression removes the version. Usually license is not changed with version bumps
#jacoco and test dependencies are excluded
-find . -type f -name "*.jar" | cut -c3- | perl -wpl -e 's/-[0-9]+(.[0-9]+)*(-([0-9a-z]+-)?SNAPSHOT)?+//g; s/\.v\d+\.jar/.jar/g;' | grep -v -e jacoco -e hdds-test-utils | sort > "$SCRIPTDIR"/$REPORT_NAME
+find . -type f -name "*.jar" | cut -c3- | perl -wpl -e 's/-[0-9]+(\.[0-9]+)*(-([0-9a-z]+-)?SNAPSHOT)?+//g; s/\.v\d+\.jar/.jar/g;' | grep -v -e jacoco -e hdds-test-utils | sort > "$SCRIPTDIR"/$REPORT_NAME
diff --git a/pom.xml b/pom.xml
index e69803f9d2..3f93e9683c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<declared.ozone.version>${ozone.version}</declared.ozone.version>
<!-- Apache Ratis version -->
- <ratis.version>2.2.0</ratis.version>
+ <ratis.version>2.3.0-94db58b-SNAPSHOT</ratis.version>
<!-- Apache Ratis thirdparty version -->
<ratis.thirdparty.version>0.7.0</ratis.thirdparty.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 05/35: HDDS-5488. [Ozone-Streaming] Add a new BlockOutputStream/KeyOutputStream to support streaming api (#2495)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3641548d5612f007c4bdb0a603deda1896b750c9
Author: Kaijie Chen <ch...@kaijie.org>
AuthorDate: Thu Aug 12 18:09:38 2021 +0800
HDDS-5488. [Ozone-Streaming] Add a new BlockOutputStream/KeyOutputStream to support streaming api (#2495)
---
hadoop-hdds/client/pom.xml | 4 +
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 5 +
.../hdds/scm/storage/BlockDataStreamOutput.java | 760 +++++++++++++++++++++
.../hdds/scm/storage/ByteBufStreamOutput.java | 58 ++
.../apache/hadoop/ozone/client/OzoneBucket.java | 19 +
.../client/io/BlockDataStreamOutputEntry.java | 294 ++++++++
.../client/io/BlockDataStreamOutputEntryPool.java | 324 +++++++++
.../ozone/client/io/KeyDataStreamOutput.java | 629 +++++++++++++++++
.../ozone/client/io/OzoneDataStreamOutput.java | 70 ++
.../ozone/client/protocol/ClientProtocol.java | 15 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 62 ++
.../client/rpc/TestBlockDataStreamOutput.java | 181 +++++
.../apache/hadoop/ozone/container/TestHelper.java | 22 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 40 +-
14 files changed, 2477 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
index 9f2116c96f..17120fa250 100644
--- a/hadoop-hdds/client/pom.xml
+++ b/hadoop-hdds/client/pom.xml
@@ -70,6 +70,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>${spotbugs.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 6982d41fbc..f37cd1c94a 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdds.tracing.TracingUtil;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.client.api.DataStreamApi;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.exceptions.GroupMismatchException;
@@ -359,4 +360,8 @@ public final class XceiverClientRatis extends XceiverClientSpi {
throw new UnsupportedOperationException(
"Operation Not supported for ratis client");
}
+
+ public DataStreamApi getDataStreamApi() {
+ return this.getClient().getDataStreamApi();
+ }
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
new file mode 100644
index 0000000000..f658df1af9
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -0,0 +1,760 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import io.netty.buffer.ByteBuf;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
+import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.XceiverClientRatis;
+import org.apache.hadoop.hdds.scm.XceiverClientReply;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.common.Checksum;
+import org.apache.hadoop.ozone.common.ChecksumData;
+import org.apache.hadoop.ozone.common.ChunkBuffer;
+import org.apache.hadoop.ozone.common.OzoneChecksumException;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.ratis.client.api.DataStreamOutput;
+import org.apache.ratis.io.StandardWriteOption;
+import org.apache.ratis.protocol.DataStreamReply;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putBlockAsync;
+
+/**
+ * An {@link ByteBufStreamOutput} used by the REST service in combination
+ * with the SCMClient to write the value of a key to a sequence
+ * of container chunks. Writes are buffered locally and periodically written to
+ * the container as a new chunk. In order to preserve the semantics that
+ * replacement of a pre-existing key is atomic, each instance of the stream has
+ * an internal unique identifier. This unique identifier and a monotonically
+ * increasing chunk index form a composite key that is used as the chunk name.
+ * After all data is written, a putKey call creates or updates the corresponding
+ * container key, and this call includes the full list of chunks that make up
+ * the key data. The list of chunks is updated all at once. Therefore, a
+ * concurrent reader never can see an intermediate state in which different
+ * chunks of data from different versions of the key data are interleaved.
+ * This class encapsulates all state management for buffering and writing
+ * through to the container.
+ */
+public class BlockDataStreamOutput implements ByteBufStreamOutput {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(BlockDataStreamOutput.class);
+ public static final String EXCEPTION_MSG =
+ "Unexpected Storage Container Exception: ";
+ private static final CompletableFuture[] EMPTY_FUTURE_ARRAY = {};
+
+ private AtomicReference<BlockID> blockID;
+
+ private final BlockData.Builder containerBlockData;
+ private XceiverClientFactory xceiverClientFactory;
+ private XceiverClientRatis xceiverClient;
+ private OzoneClientConfig config;
+
+ private int chunkIndex;
+ private final AtomicLong chunkOffset = new AtomicLong();
+ private final BufferPool bufferPool;
+ // The IOException will be set by response handling thread in case there is an
+ // exception received in the response. If the exception is set, the next
+ // request will fail upfront.
+ private final AtomicReference<IOException> ioException;
+ private final ExecutorService responseExecutor;
+
+ // the effective length of data flushed so far
+ private long totalDataFlushedLength;
+
+ // effective data write attempted so far for the block
+ private long writtenDataLength;
+
+ // List containing buffers for which the putBlock call will
+ // update the length in the datanodes. This list will just maintain
+ // references to the buffers in the BufferPool which will be cleared
+ // when the watchForCommit acknowledges a putBlock logIndex has been
+ // committed on all datanodes. This list will be a place holder for buffers
+ // which got written between successive putBlock calls.
+ private List<ChunkBuffer> bufferList;
+
+ // This object will maintain the commitIndexes and byteBufferList in order
+ // Also, corresponding to the logIndex, the corresponding list of buffers will
+ // be released from the buffer pool.
+ private final CommitWatcher commitWatcher;
+
+ private final List<DatanodeDetails> failedServers;
+ private final Checksum checksum;
+
+ //number of buffers used before doing a flush/putBlock.
+ private int flushPeriod;
+ //bytes remaining to write in the current buffer.
+ private int currentBufferRemaining;
+ //current buffer allocated to write
+ private ChunkBuffer currentBuffer;
+ private final Token<? extends TokenIdentifier> token;
+ private final DataStreamOutput out;
+ private CompletableFuture<DataStreamReply> dataStreamCloseReply;
+ private List<CompletableFuture<DataStreamReply>> futures = new ArrayList<>();
+ private final long syncSize = 0; // TODO: disk sync is disabled for now
+ private long syncPosition = 0;
+
+ /**
+ * Creates a new BlockDataStreamOutput.
+ *
+ * @param blockID block ID
+ * @param xceiverClientManager client manager that controls client
+ * @param pipeline pipeline where block will be written
+ * @param bufferPool pool of buffers
+ */
+ public BlockDataStreamOutput(
+ BlockID blockID,
+ XceiverClientFactory xceiverClientManager,
+ Pipeline pipeline,
+ BufferPool bufferPool,
+ OzoneClientConfig config,
+ Token<? extends TokenIdentifier> token
+ ) throws IOException {
+ this.xceiverClientFactory = xceiverClientManager;
+ this.config = config;
+ this.blockID = new AtomicReference<>(blockID);
+ KeyValue keyValue =
+ KeyValue.newBuilder().setKey("TYPE").setValue("KEY").build();
+ this.containerBlockData =
+ BlockData.newBuilder().setBlockID(blockID.getDatanodeBlockIDProtobuf())
+ .addMetadata(keyValue);
+ this.xceiverClient =
+ (XceiverClientRatis)xceiverClientManager.acquireClient(pipeline);
+ // Alternatively, stream setup can be delayed till the first chunk write.
+ this.out = setupStream();
+ this.bufferPool = bufferPool;
+ this.token = token;
+
+ //number of buffers used before doing a flush
+ refreshCurrentBuffer(bufferPool);
+ flushPeriod = (int) (config.getStreamBufferFlushSize() / config
+ .getStreamBufferSize());
+
+ Preconditions
+ .checkArgument(
+ (long) flushPeriod * config.getStreamBufferSize() == config
+ .getStreamBufferFlushSize());
+
+ // A single thread executor handle the responses of async requests
+ responseExecutor = Executors.newSingleThreadExecutor();
+ commitWatcher = new CommitWatcher(bufferPool, xceiverClient);
+ bufferList = null;
+ totalDataFlushedLength = 0;
+ writtenDataLength = 0;
+ failedServers = new ArrayList<>(0);
+ ioException = new AtomicReference<>(null);
+ checksum = new Checksum(config.getChecksumType(),
+ config.getBytesPerChecksum());
+ }
+
+ private DataStreamOutput setupStream() throws IOException {
+ // Execute a dummy WriteChunk request to get the path of the target file,
+ // but does NOT write any data to it.
+ ContainerProtos.WriteChunkRequestProto.Builder writeChunkRequest =
+ ContainerProtos.WriteChunkRequestProto.newBuilder()
+ .setBlockID(blockID.get().getDatanodeBlockIDProtobuf());
+
+ String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
+ ContainerProtos.ContainerCommandRequestProto.Builder builder =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder()
+ .setCmdType(ContainerProtos.Type.StreamInit)
+ .setContainerID(blockID.get().getContainerID())
+ .setDatanodeUuid(id).setWriteChunk(writeChunkRequest);
+
+ ContainerCommandRequestMessage message =
+ ContainerCommandRequestMessage.toMessage(builder.build(), null);
+
+ return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
+ .stream(message.getContent().asReadOnlyByteBuffer());
+ }
+
+ private void refreshCurrentBuffer(BufferPool pool) {
+ currentBuffer = pool.getCurrentBuffer();
+ currentBufferRemaining =
+ currentBuffer != null ? currentBuffer.remaining() : 0;
+ }
+
+ public BlockID getBlockID() {
+ return blockID.get();
+ }
+
+ public long getTotalAckDataLength() {
+ return commitWatcher.getTotalAckDataLength();
+ }
+
+ public long getWrittenDataLength() {
+ return writtenDataLength;
+ }
+
+ public List<DatanodeDetails> getFailedServers() {
+ return failedServers;
+ }
+
+ @VisibleForTesting
+ public XceiverClientRatis getXceiverClient() {
+ return xceiverClient;
+ }
+
+ @VisibleForTesting
+ public long getTotalDataFlushedLength() {
+ return totalDataFlushedLength;
+ }
+
+ @VisibleForTesting
+ public BufferPool getBufferPool() {
+ return bufferPool;
+ }
+
+ public IOException getIoException() {
+ return ioException.get();
+ }
+
+ @VisibleForTesting
+ public Map<Long, List<ChunkBuffer>> getCommitIndex2flushedDataMap() {
+ return commitWatcher.getCommitIndex2flushedDataMap();
+ }
+
+ @Override
+ public void write(ByteBuf b) throws IOException {
+ checkOpen();
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ int off = b.readerIndex();
+ int len = b.readableBytes();
+
+ while (len > 0) {
+ allocateNewBufferIfNeeded();
+ final int writeLen = Math.min(currentBufferRemaining, len);
+ // TODO: avoid buffer copy here
+ currentBuffer.put(b.nioBuffer(off, writeLen));
+ currentBufferRemaining -= writeLen;
+ writeChunkIfNeeded();
+ off += writeLen;
+ len -= writeLen;
+ writtenDataLength += writeLen;
+ doFlushOrWatchIfNeeded();
+ }
+ }
+
+ private void writeChunkIfNeeded() throws IOException {
+ if (currentBufferRemaining == 0) {
+ writeChunk(currentBuffer);
+ }
+ }
+
+ private void doFlushOrWatchIfNeeded() throws IOException {
+ if (currentBufferRemaining == 0) {
+ if (bufferPool.getNumberOfUsedBuffers() % flushPeriod == 0) {
+ updateFlushLength();
+ executePutBlock(false, false);
+ }
+ // Data in the bufferPool can not exceed streamBufferMaxSize
+ if (bufferPool.getNumberOfUsedBuffers() == bufferPool.getCapacity()) {
+ handleFullBuffer();
+ }
+ }
+ }
+
+ private void allocateNewBufferIfNeeded() {
+ if (currentBufferRemaining == 0) {
+ currentBuffer = bufferPool.allocateBuffer(config.getBufferIncrement());
+ currentBufferRemaining = currentBuffer.remaining();
+ }
+ }
+
+ private void updateFlushLength() {
+ totalDataFlushedLength = writtenDataLength;
+ }
+
+ private boolean isBufferPoolFull() {
+ return bufferPool.computeBufferData() == config.getStreamBufferMaxSize();
+ }
+
+ /**
+ * Will be called on the retryPath in case closedContainerException/
+ * TimeoutException.
+ * @param len length of data to write
+ * @throws IOException if error occurred
+ */
+
+ // In this case, the data is already cached in the currentBuffer.
+ public void writeOnRetry(long len) throws IOException {
+ if (len == 0) {
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrying write length {} for blockID {}", len, blockID);
+ }
+ Preconditions.checkArgument(len <= config.getStreamBufferMaxSize());
+ int count = 0;
+ while (len > 0) {
+ ChunkBuffer buffer = bufferPool.getBuffer(count);
+ long writeLen = Math.min(buffer.position(), len);
+ if (!buffer.hasRemaining()) {
+ writeChunk(buffer);
+ }
+ len -= writeLen;
+ count++;
+ writtenDataLength += writeLen;
+ // we should not call isBufferFull/shouldFlush here.
+ // The buffer might already be full as whole data is already cached in
+ // the buffer. We should just validate
+ // if we wrote data of size streamBufferMaxSize/streamBufferFlushSize to
+ // call for handling full buffer/flush buffer condition.
+ if (writtenDataLength % config.getStreamBufferFlushSize() == 0) {
+ // reset the position to zero as now we will be reading the
+ // next buffer in the list
+ updateFlushLength();
+ executePutBlock(false, false);
+ }
+ if (writtenDataLength == config.getStreamBufferMaxSize()) {
+ handleFullBuffer();
+ }
+ }
+ }
+
+ /**
+ * This is a blocking call. It will wait for the flush till the commit index
+ * at the head of the commitIndex2flushedDataMap gets replicated to all or
+ * majority.
+ * @throws IOException
+ */
+ private void handleFullBuffer() throws IOException {
+ try {
+ checkOpen();
+ if (!commitWatcher.getFutureMap().isEmpty()) {
+ waitOnFlushFutures();
+ }
+ } catch (ExecutionException e) {
+ handleExecutionException(e);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ handleInterruptedException(ex, true);
+ }
+ watchForCommit(true);
+ }
+
+
+ // It may happen that once the exception is encountered , we still might
+ // have successfully flushed up to a certain index. Make sure the buffers
+ // only contain data which have not been sufficiently replicated
+ private void adjustBuffersOnException() {
+ commitWatcher.releaseBuffersOnException();
+ refreshCurrentBuffer(bufferPool);
+ }
+
+ /**
+ * calls watchForCommit API of the Ratis Client. For Standalone client,
+ * it is a no op.
+ * @param bufferFull flag indicating whether bufferFull condition is hit or
+ * its called as part flush/close
+ * @return minimum commit index replicated to all nodes
+ * @throws IOException IOException in case watch gets timed out
+ */
+ private void watchForCommit(boolean bufferFull) throws IOException {
+ checkOpen();
+ try {
+ XceiverClientReply reply = bufferFull ?
+ commitWatcher.watchOnFirstIndex() : commitWatcher.watchOnLastIndex();
+ if (reply != null) {
+ List<DatanodeDetails> dnList = reply.getDatanodes();
+ if (!dnList.isEmpty()) {
+ Pipeline pipe = xceiverClient.getPipeline();
+
+ LOG.warn("Failed to commit BlockId {} on {}. Failed nodes: {}",
+ blockID, pipe, dnList);
+ failedServers.addAll(dnList);
+ }
+ }
+ } catch (IOException ioe) {
+ setIoException(ioe);
+ throw getIoException();
+ }
+ refreshCurrentBuffer(bufferPool);
+
+ }
+
+ /**
+ * @param close whether putBlock is happening as part of closing the stream
+ * @param force true if no data was written since most recent putBlock and
+ * stream is being closed
+ */
+ private CompletableFuture<ContainerProtos.
+ ContainerCommandResponseProto> executePutBlock(boolean close,
+ boolean force) throws IOException {
+ checkOpen();
+ long flushPos = totalDataFlushedLength;
+ final List<ChunkBuffer> byteBufferList;
+ if (!force) {
+ Preconditions.checkNotNull(bufferList);
+ byteBufferList = bufferList;
+ bufferList = null;
+ Preconditions.checkNotNull(byteBufferList);
+ } else {
+ byteBufferList = null;
+ }
+
+ try {
+ CompletableFuture.allOf(futures.toArray(EMPTY_FUTURE_ARRAY)).get();
+ } catch (Exception e) {
+ LOG.warn("Failed to write all chunks through stream: " + e);
+ throw new IOException(e);
+ }
+ if (close) {
+ dataStreamCloseReply = out.closeAsync();
+ }
+
+ CompletableFuture<ContainerProtos.
+ ContainerCommandResponseProto> flushFuture = null;
+ try {
+ BlockData blockData = containerBlockData.build();
+ XceiverClientReply asyncReply =
+ putBlockAsync(xceiverClient, blockData, close, token);
+ CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
+ asyncReply.getResponse();
+ flushFuture = future.thenApplyAsync(e -> {
+ try {
+ validateResponse(e);
+ } catch (IOException sce) {
+ throw new CompletionException(sce);
+ }
+ // if the ioException is not set, putBlock is successful
+ if (getIoException() == null && !force) {
+ BlockID responseBlockID = BlockID.getFromProtobuf(
+ e.getPutBlock().getCommittedBlockLength().getBlockID());
+ Preconditions.checkState(blockID.get().getContainerBlockID()
+ .equals(responseBlockID.getContainerBlockID()));
+ // updates the bcsId of the block
+ blockID.set(responseBlockID);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Adding index " + asyncReply.getLogIndex() + " commitMap size "
+ + commitWatcher.getCommitInfoMapSize() + " flushLength "
+ + flushPos + " numBuffers " + byteBufferList.size()
+ + " blockID " + blockID + " bufferPool size" + bufferPool
+ .getSize() + " currentBufferIndex " + bufferPool
+ .getCurrentBufferIndex());
+ }
+ // for standalone protocol, logIndex will always be 0.
+ commitWatcher
+ .updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
+ }
+ return e;
+ }, responseExecutor).exceptionally(e -> {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("putBlock failed for blockID {} with exception {}",
+ blockID, e.getLocalizedMessage());
+ }
+ CompletionException ce = new CompletionException(e);
+ setIoException(ce);
+ throw ce;
+ });
+ } catch (IOException | ExecutionException e) {
+ throw new IOException(EXCEPTION_MSG + e.toString(), e);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ handleInterruptedException(ex, false);
+ }
+ commitWatcher.getFutureMap().put(flushPos, flushFuture);
+ return flushFuture;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (xceiverClientFactory != null && xceiverClient != null
+ && bufferPool != null && bufferPool.getSize() > 0
+ && (!config.isStreamBufferFlushDelay() ||
+ writtenDataLength - totalDataFlushedLength
+ >= config.getStreamBufferSize())) {
+ try {
+ handleFlush(false);
+ } catch (ExecutionException e) {
+ // just set the exception here as well in order to maintain sanctity of
+ // ioException field
+ handleExecutionException(e);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ handleInterruptedException(ex, true);
+ }
+ }
+ }
+
+ private void writeChunk(ChunkBuffer buffer)
+ throws IOException {
+ // This data in the buffer will be pushed to datanode and a reference will
+ // be added to the bufferList. Once putBlock gets executed, this list will
+ // be marked null. Hence, during first writeChunk call after every putBlock
+ // call or during the first call to writeChunk here, the list will be null.
+
+ if (bufferList == null) {
+ bufferList = new ArrayList<>();
+ }
+ bufferList.add(buffer);
+ writeChunkToContainer(buffer.duplicate(0, buffer.position()));
+ }
+
+ /**
+ * @param close whether the flush is happening as part of closing the stream
+ */
+ private void handleFlush(boolean close)
+ throws IOException, InterruptedException, ExecutionException {
+ checkOpen();
+ // flush the last chunk data residing on the currentBuffer
+ if (totalDataFlushedLength < writtenDataLength) {
+ refreshCurrentBuffer(bufferPool);
+ Preconditions.checkArgument(currentBuffer.position() > 0);
+ if (currentBuffer.hasRemaining()) {
+ writeChunk(currentBuffer);
+ }
+ // This can be a partially filled chunk. Since we are flushing the buffer
+ // here, we just limit this buffer to the current position. So that next
+ // write will happen in new buffer
+ updateFlushLength();
+ executePutBlock(close, false);
+ } else if (close) {
+ // forcing an "empty" putBlock if stream is being closed without new
+ // data since latest flush - we need to send the "EOF" flag
+ executePutBlock(true, true);
+ }
+ waitOnFlushFutures();
+ watchForCommit(false);
+ // just check again if the exception is hit while waiting for the
+ // futures to ensure flush has indeed succeeded
+
+ // irrespective of whether the commitIndex2flushedDataMap is empty
+ // or not, ensure there is no exception set
+ checkOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (xceiverClientFactory != null && xceiverClient != null
+ && bufferPool != null && bufferPool.getSize() > 0) {
+ try {
+ handleFlush(true);
+ dataStreamCloseReply.get();
+ } catch (ExecutionException e) {
+ handleExecutionException(e);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ handleInterruptedException(ex, true);
+ } finally {
+ cleanup(false);
+ }
+ // TODO: Turn the below buffer empty check on when Standalone pipeline
+ // is removed in the write path in tests
+ // Preconditions.checkArgument(buffer.position() == 0);
+ // bufferPool.checkBufferPoolEmpty();
+
+ }
+ }
+
+ private void waitOnFlushFutures()
+ throws InterruptedException, ExecutionException {
+ CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(
+ commitWatcher.getFutureMap().values().toArray(
+ new CompletableFuture[commitWatcher.getFutureMap().size()]));
+ // wait for all the transactions to complete
+ combinedFuture.get();
+ }
+
+ private void validateResponse(
+ ContainerProtos.ContainerCommandResponseProto responseProto)
+ throws IOException {
+ try {
+ // if the ioException is already set, it means a prev request has failed
+ // just throw the exception. The current operation will fail with the
+ // original error
+ IOException exception = getIoException();
+ if (exception != null) {
+ throw exception;
+ }
+ ContainerProtocolCalls.validateContainerResponse(responseProto);
+ } catch (StorageContainerException sce) {
+ setIoException(sce);
+ throw sce;
+ }
+ }
+
+
+ private void setIoException(Exception e) {
+ IOException ioe = getIoException();
+ if (ioe == null) {
+ IOException exception = new IOException(EXCEPTION_MSG + e.toString(), e);
+ ioException.compareAndSet(null, exception);
+ } else {
+ LOG.debug("Previous request had already failed with " + ioe.toString()
+ + " so subsequent request also encounters"
+ + " Storage Container Exception ", e);
+ }
+ }
+
+ public void cleanup(boolean invalidateClient) {
+ if (xceiverClientFactory != null) {
+ xceiverClientFactory.releaseClient(xceiverClient, invalidateClient);
+ }
+ xceiverClientFactory = null;
+ xceiverClient = null;
+ commitWatcher.cleanup();
+ if (bufferList != null) {
+ bufferList.clear();
+ }
+ bufferList = null;
+ responseExecutor.shutdown();
+ }
+
+ /**
+ * Checks if the stream is open or exception has occurred.
+ * If not, throws an exception.
+ *
+ * @throws IOException if stream is closed
+ */
+ private void checkOpen() throws IOException {
+ if (isClosed()) {
+ throw new IOException("BlockDataStreamOutput has been closed.");
+ } else if (getIoException() != null) {
+ adjustBuffersOnException();
+ throw getIoException();
+ }
+ }
+
+ public boolean isClosed() {
+ return xceiverClient == null;
+ }
+
+ private boolean needSync(long position) {
+ if (syncSize > 0) {
+ // TODO: or position >= fileLength
+ if (position - syncPosition >= syncSize) {
+ syncPosition = position;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Writes buffered data as a new chunk to the container and saves chunk
+ * information to be used later in putKey call.
+ *
+ * @throws IOException if there is an I/O error while performing the call
+ * @throws OzoneChecksumException if there is an error while computing
+ * checksum
+ */
+ private void writeChunkToContainer(ChunkBuffer chunk) throws IOException {
+ int effectiveChunkSize = chunk.remaining();
+ final long offset = chunkOffset.getAndAdd(effectiveChunkSize);
+ final ByteString data = chunk.toByteString(
+ bufferPool.byteStringConversion());
+ ChecksumData checksumData = checksum.computeChecksum(chunk);
+ ChunkInfo chunkInfo = ChunkInfo.newBuilder()
+ .setChunkName(blockID.get().getLocalID() + "_chunk_" + ++chunkIndex)
+ .setOffset(offset)
+ .setLen(effectiveChunkSize)
+ .setChecksumData(checksumData.getProtoBufMessage())
+ .build();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Writing chunk {} length {} at offset {}",
+ chunkInfo.getChunkName(), effectiveChunkSize, offset);
+ }
+
+ CompletableFuture<DataStreamReply> future =
+ (needSync(offset + effectiveChunkSize) ?
+ out.writeAsync(data.asReadOnlyByteBuffer(), StandardWriteOption.SYNC) :
+ out.writeAsync(data.asReadOnlyByteBuffer()))
+ .whenCompleteAsync((r, e) -> {
+ if (e != null || !r.isSuccess()) {
+ if (e == null) {
+ e = new IOException("result is not success");
+ }
+ String msg = "Failed to write chunk " + chunkInfo.getChunkName() +
+ " " + "into block " + blockID;
+ LOG.debug("{}, exception: {}", msg, e.getLocalizedMessage());
+ CompletionException ce = new CompletionException(msg, e);
+ setIoException(ce);
+ throw ce;
+ }
+ }, responseExecutor);
+
+ futures.add(future);
+ containerBlockData.addChunks(chunkInfo);
+ }
+
+ @VisibleForTesting
+ public void setXceiverClient(XceiverClientRatis xceiverClient) {
+ this.xceiverClient = xceiverClient;
+ }
+
+ /**
+ * Handles InterruptedExecution.
+ *
+ * @param ex
+ * @param processExecutionException is optional, if passed as TRUE, then
+ * handle ExecutionException else skip it.
+ * @throws IOException
+ */
+ private void handleInterruptedException(Exception ex,
+ boolean processExecutionException)
+ throws IOException {
+ LOG.error("Command execution was interrupted.");
+ if(processExecutionException) {
+ handleExecutionException(ex);
+ } else {
+ throw new IOException(EXCEPTION_MSG + ex.toString(), ex);
+ }
+ }
+
+ /**
+ * Handles ExecutionException by adjusting buffers.
+ * @param ex
+ * @throws IOException
+ */
+ private void handleExecutionException(Exception ex) throws IOException {
+ setIoException(ex);
+ adjustBuffersOnException();
+ throw getIoException();
+ }
+}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java
new file mode 100644
index 0000000000..7f40737b70
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufStreamOutput.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import io.netty.buffer.ByteBuf;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+* This interface is for writing an output stream of ByteBuffers.
+* An ByteBufStreamOutput accepts Netty ByteBuf and sends them to some sink.
+*/
+public interface ByteBufStreamOutput extends Closeable {
+ /**
+ * Try to write all the bytes in ByteBuf b to DataStream.
+ *
+ * @param b the data.
+ * @exception IOException if an I/O error occurs.
+ */
+ void write(ByteBuf b) throws IOException;
+
+ /**
+ * Try to write the [off:off + len) slice in ByteBuf b to DataStream.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException if an I/O error occurs.
+ */
+ default void write(ByteBuf b, int off, int len) throws IOException {
+ write(b.slice(off, len));
+ }
+
+ /**
+ * Flushes this DataStream output and forces any buffered output bytes
+ * to be written out.
+ *
+ * @exception IOException if an I/O error occurs.
+ */
+ void flush() throws IOException;
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 9ae41855f6..89fe02d678 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
@@ -596,6 +597,24 @@ public class OzoneBucket extends WithMetadata {
.createKey(volumeName, name, key, size, replicationConfig, keyMetadata);
}
+ /**
+ * Creates a new key in the bucket.
+ *
+ * @param key Name of the key to be created.
+ * @param size Size of the data the key will point to.
+ * @param replicationConfig Replication configuration.
+ * @return OzoneDataStreamOutput to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneDataStreamOutput createStreamKey(String key, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata)
+ throws IOException {
+ return proxy
+ .createStreamKey(volumeName, name, key, size, replicationConfig,
+ keyMetadata);
+ }
+
/**
* Reads an existing key from the bucket.
*
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
new file mode 100644
index 0000000000..6954742601
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client.io;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.netty.buffer.ByteBuf;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.BufferPool;
+import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Helper class used inside {@link BlockDataStreamOutput}.
+ * */
+public final class BlockDataStreamOutputEntry
+ implements ByteBufStreamOutput {
+
+ private final OzoneClientConfig config;
+ private ByteBufStreamOutput byteBufStreamOutput;
+ private BlockID blockID;
+ private final String key;
+ private final XceiverClientFactory xceiverClientManager;
+ private final Pipeline pipeline;
+ // total number of bytes that should be written to this stream
+ private final long length;
+ // the current position of this stream 0 <= currentPosition < length
+ private long currentPosition;
+ private final Token<OzoneBlockTokenIdentifier> token;
+
+ private BufferPool bufferPool;
+
+ @SuppressWarnings({"parameternumber", "squid:S00107"})
+ private BlockDataStreamOutputEntry(
+ BlockID blockID, String key,
+ XceiverClientFactory xceiverClientManager,
+ Pipeline pipeline,
+ long length,
+ BufferPool bufferPool,
+ Token<OzoneBlockTokenIdentifier> token,
+ OzoneClientConfig config
+ ) {
+ this.config = config;
+ this.byteBufStreamOutput = null;
+ this.blockID = blockID;
+ this.key = key;
+ this.xceiverClientManager = xceiverClientManager;
+ this.pipeline = pipeline;
+ this.token = token;
+ this.length = length;
+ this.currentPosition = 0;
+ this.bufferPool = bufferPool;
+ }
+
+ long getLength() {
+ return length;
+ }
+
+ Token<OzoneBlockTokenIdentifier> getToken() {
+ return token;
+ }
+
+ long getRemaining() {
+ return length - currentPosition;
+ }
+
+ /**
+ * BlockDataStreamOutput is initialized in this function. This makes sure that
+ * xceiverClient initialization is not done during preallocation and only
+ * done when data is written.
+ * @throws IOException if xceiverClient initialization fails
+ */
+ private void checkStream() throws IOException {
+ if (this.byteBufStreamOutput == null) {
+ this.byteBufStreamOutput =
+ new BlockDataStreamOutput(blockID, xceiverClientManager,
+ pipeline, bufferPool, config, token);
+ }
+ }
+
+ @Override
+ public void write(ByteBuf b) throws IOException {
+ checkStream();
+ final int len = b.readableBytes();
+ byteBufStreamOutput.write(b);
+ this.currentPosition += len;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (this.byteBufStreamOutput != null) {
+ this.byteBufStreamOutput.flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (this.byteBufStreamOutput != null) {
+ this.byteBufStreamOutput.close();
+ // after closing the chunkOutPutStream, blockId would have been
+ // reconstructed with updated bcsId
+ this.blockID =
+ ((BlockDataStreamOutput) byteBufStreamOutput).getBlockID();
+ }
+ }
+
+ boolean isClosed() {
+ if (byteBufStreamOutput != null) {
+ return ((BlockDataStreamOutput) byteBufStreamOutput).isClosed();
+ }
+ return false;
+ }
+
+ long getTotalAckDataLength() {
+ if (byteBufStreamOutput != null) {
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufStreamOutput;
+ blockID = out.getBlockID();
+ return out.getTotalAckDataLength();
+ } else {
+ // For a pre allocated block for which no write has been initiated,
+ // the ByteBufStreamOutput will be null here.
+ // In such cases, the default blockCommitSequenceId will be 0
+ return 0;
+ }
+ }
+
+ Collection<DatanodeDetails> getFailedServers() {
+ if (byteBufStreamOutput != null) {
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufStreamOutput;
+ return out.getFailedServers();
+ }
+ return Collections.emptyList();
+ }
+
+ long getWrittenDataLength() {
+ if (byteBufStreamOutput != null) {
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufStreamOutput;
+ return out.getWrittenDataLength();
+ } else {
+ // For a pre allocated block for which no write has been initiated,
+ // the ByteBufStreamOutput will be null here.
+ // In such cases, the default blockCommitSequenceId will be 0
+ return 0;
+ }
+ }
+
+ void cleanup(boolean invalidateClient) throws IOException {
+ checkStream();
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufStreamOutput;
+ out.cleanup(invalidateClient);
+
+ }
+
+ void writeOnRetry(long len) throws IOException {
+ checkStream();
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufStreamOutput;
+ out.writeOnRetry(len);
+ this.currentPosition += len;
+
+ }
+
+ /**
+ * Builder class for BlockDataStreamOutputEntry.
+ * */
+ public static class Builder {
+
+ private BlockID blockID;
+ private String key;
+ private XceiverClientFactory xceiverClientManager;
+ private Pipeline pipeline;
+ private long length;
+ private BufferPool bufferPool;
+ private Token<OzoneBlockTokenIdentifier> token;
+ private OzoneClientConfig config;
+
+ public Builder setBlockID(BlockID bID) {
+ this.blockID = bID;
+ return this;
+ }
+
+ public Builder setKey(String keys) {
+ this.key = keys;
+ return this;
+ }
+
+ public Builder setXceiverClientManager(
+ XceiverClientFactory
+ xClientManager) {
+ this.xceiverClientManager = xClientManager;
+ return this;
+ }
+
+ public Builder setPipeline(Pipeline ppln) {
+ this.pipeline = ppln;
+ return this;
+ }
+
+
+ public Builder setLength(long len) {
+ this.length = len;
+ return this;
+ }
+
+
+ public Builder setBufferPool(BufferPool pool) {
+ this.bufferPool = pool;
+ return this;
+ }
+
+ public Builder setConfig(OzoneClientConfig clientConfig) {
+ this.config = clientConfig;
+ return this;
+ }
+
+ public Builder setToken(Token<OzoneBlockTokenIdentifier> bToken) {
+ this.token = bToken;
+ return this;
+ }
+
+ public BlockDataStreamOutputEntry build() {
+ return new BlockDataStreamOutputEntry(blockID,
+ key,
+ xceiverClientManager,
+ pipeline,
+ length,
+ bufferPool,
+ token, config);
+ }
+ }
+
+ @VisibleForTesting
+ public ByteBufStreamOutput getByteBufStreamOutput() {
+ return byteBufStreamOutput;
+ }
+
+ public BlockID getBlockID() {
+ return blockID;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public XceiverClientFactory getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
+ public Pipeline getPipeline() {
+ return pipeline;
+ }
+
+ public long getCurrentPosition() {
+ return currentPosition;
+ }
+
+ public BufferPool getBufferPool() {
+ return bufferPool;
+ }
+
+ public void setCurrentPosition(long curPosition) {
+ this.currentPosition = curPosition;
+ }
+}
+
+
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
new file mode 100644
index 0000000000..94c505f2af
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -0,0 +1,324 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client.io;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ByteStringConversion;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.BufferPool;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * This class manages the stream entries list and handles block allocation
+ * from OzoneManager.
+ */
+public class BlockDataStreamOutputEntryPool {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(BlockDataStreamOutputEntryPool.class);
+
+ private final List<BlockDataStreamOutputEntry> streamEntries;
+ private final OzoneClientConfig config;
+ private int currentStreamIndex;
+ private final OzoneManagerProtocol omClient;
+ private final OmKeyArgs keyArgs;
+ private final XceiverClientFactory xceiverClientFactory;
+ private final String requestID;
+ private final BufferPool bufferPool;
+ private OmMultipartCommitUploadPartInfo commitUploadPartInfo;
+ private final long openID;
+ private final ExcludeList excludeList;
+
+ @SuppressWarnings({"parameternumber", "squid:S00107"})
+ public BlockDataStreamOutputEntryPool(
+ OzoneClientConfig config,
+ OzoneManagerProtocol omClient,
+ String requestId, ReplicationConfig replicationConfig,
+ String uploadID, int partNumber,
+ boolean isMultipart, OmKeyInfo info,
+ boolean unsafeByteBufferConversion,
+ XceiverClientFactory xceiverClientFactory, long openID
+ ) {
+ this.config = config;
+ this.xceiverClientFactory = xceiverClientFactory;
+ streamEntries = new ArrayList<>();
+ currentStreamIndex = 0;
+ this.omClient = omClient;
+ this.keyArgs = new OmKeyArgs.Builder().setVolumeName(info.getVolumeName())
+ .setBucketName(info.getBucketName()).setKeyName(info.getKeyName())
+ .setReplicationConfig(replicationConfig).setDataSize(info.getDataSize())
+ .setIsMultipartKey(isMultipart).setMultipartUploadID(uploadID)
+ .setMultipartUploadPartNumber(partNumber).build();
+ this.requestID = requestId;
+ this.openID = openID;
+ this.excludeList = new ExcludeList();
+
+ this.bufferPool =
+ new BufferPool(config.getStreamBufferSize(),
+ (int) (config.getStreamBufferMaxSize() / config
+ .getStreamBufferSize()),
+ ByteStringConversion
+ .createByteBufferConversion(unsafeByteBufferConversion));
+ }
+
+ /**
+ * A constructor for testing purpose only.
+ *
+ * @see KeyDataStreamOutput#KeyDataStreamOutput()
+ */
+ @VisibleForTesting
+ BlockDataStreamOutputEntryPool() {
+ streamEntries = new ArrayList<>();
+ omClient = null;
+ keyArgs = null;
+ xceiverClientFactory = null;
+ config =
+ new OzoneConfiguration().getObject(OzoneClientConfig.class);
+ config.setStreamBufferSize(0);
+ config.setStreamBufferMaxSize(0);
+ config.setStreamBufferFlushSize(0);
+ config.setStreamBufferFlushDelay(false);
+ requestID = null;
+ int chunkSize = 0;
+ bufferPool = new BufferPool(chunkSize, 1);
+
+ currentStreamIndex = 0;
+ openID = -1;
+ excludeList = new ExcludeList();
+ }
+
+ /**
+ * When a key is opened, it is possible that there are some blocks already
+ * allocated to it for this open session. In this case, to make use of these
+ * blocks, we need to add these blocks to stream entries. But, a key's version
+ * also includes blocks from previous versions, we need to avoid adding these
+ * old blocks to stream entries, because these old blocks should not be picked
+ * for write. To do this, the following method checks that, only those
+ * blocks created in this particular open version are added to stream entries.
+ *
+ * @param version the set of blocks that are pre-allocated.
+ * @param openVersion the version corresponding to the pre-allocation.
+ * @throws IOException
+ */
+ public void addPreallocateBlocks(OmKeyLocationInfoGroup version,
+ long openVersion) throws IOException {
+ // server may return any number of blocks, (0 to any)
+ // only the blocks allocated in this open session (block createVersion
+ // equals to open session version)
+ for (OmKeyLocationInfo subKeyInfo : version.getLocationList(openVersion)) {
+ addKeyLocationInfo(subKeyInfo);
+ }
+ }
+
+ private void addKeyLocationInfo(OmKeyLocationInfo subKeyInfo) {
+ Preconditions.checkNotNull(subKeyInfo.getPipeline());
+ BlockDataStreamOutputEntry.Builder builder =
+ new BlockDataStreamOutputEntry.Builder()
+ .setBlockID(subKeyInfo.getBlockID())
+ .setKey(keyArgs.getKeyName())
+ .setXceiverClientManager(xceiverClientFactory)
+ .setPipeline(subKeyInfo.getPipeline())
+ .setConfig(config)
+ .setLength(subKeyInfo.getLength())
+ .setBufferPool(bufferPool)
+ .setToken(subKeyInfo.getToken());
+ streamEntries.add(builder.build());
+ }
+
+ public List<OmKeyLocationInfo> getLocationInfoList() {
+ List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
+ for (BlockDataStreamOutputEntry streamEntry : streamEntries) {
+ long length = streamEntry.getCurrentPosition();
+
+ // Commit only those blocks to OzoneManager which are not empty
+ if (length != 0) {
+ OmKeyLocationInfo info =
+ new OmKeyLocationInfo.Builder().setBlockID(streamEntry.getBlockID())
+ .setLength(streamEntry.getCurrentPosition()).setOffset(0)
+ .setToken(streamEntry.getToken())
+ .setPipeline(streamEntry.getPipeline()).build();
+ locationInfoList.add(info);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "block written " + streamEntry.getBlockID() + ", length " + length
+ + " bcsID " + streamEntry.getBlockID()
+ .getBlockCommitSequenceId());
+ }
+ }
+ return locationInfoList;
+ }
+
+ /**
+ * Discards the subsequent pre allocated blocks and removes the streamEntries
+ * from the streamEntries list for the container which is closed.
+ * @param containerID id of the closed container
+ * @param pipelineId id of the associated pipeline
+ */
+ void discardPreallocatedBlocks(long containerID, PipelineID pipelineId) {
+ // currentStreamIndex < streamEntries.size() signifies that, there are still
+ // pre allocated blocks available.
+
+ // This will be called only to discard the next subsequent unused blocks
+ // in the streamEntryList.
+ if (currentStreamIndex + 1 < streamEntries.size()) {
+ ListIterator<BlockDataStreamOutputEntry> streamEntryIterator =
+ streamEntries.listIterator(currentStreamIndex + 1);
+ while (streamEntryIterator.hasNext()) {
+ BlockDataStreamOutputEntry streamEntry = streamEntryIterator.next();
+ Preconditions.checkArgument(streamEntry.getCurrentPosition() == 0);
+ if ((streamEntry.getPipeline().getId().equals(pipelineId)) ||
+ (containerID != -1 &&
+ streamEntry.getBlockID().getContainerID() == containerID)) {
+ streamEntryIterator.remove();
+ }
+ }
+ }
+ }
+
+ List<BlockDataStreamOutputEntry> getStreamEntries() {
+ return streamEntries;
+ }
+
+ XceiverClientFactory getXceiverClientFactory() {
+ return xceiverClientFactory;
+ }
+
+ String getKeyName() {
+ return keyArgs.getKeyName();
+ }
+
+ long getKeyLength() {
+ return streamEntries.stream().mapToLong(
+ BlockDataStreamOutputEntry::getCurrentPosition).sum();
+ }
+ /**
+ * Contact OM to get a new block. Set the new block with the index (e.g.
+ * first block has index = 0, second has index = 1 etc.)
+ *
+ * The returned block is made to new BlockDataStreamOutputEntry to write.
+ *
+ * @throws IOException
+ */
+ private void allocateNewBlock() throws IOException {
+ if (!excludeList.isEmpty()) {
+ LOG.debug("Allocating block with {}", excludeList);
+ }
+ OmKeyLocationInfo subKeyInfo =
+ omClient.allocateBlock(keyArgs, openID, excludeList);
+ addKeyLocationInfo(subKeyInfo);
+ }
+
+
+ void commitKey(long offset) throws IOException {
+ if (keyArgs != null) {
+ // in test, this could be null
+ long length = getKeyLength();
+ Preconditions.checkArgument(offset == length);
+ keyArgs.setDataSize(length);
+ keyArgs.setLocationInfoList(getLocationInfoList());
+ // When the key is multipart upload part file upload, we should not
+ // commit the key, as this is not an actual key, this is a just a
+ // partial key of a large file.
+ if (keyArgs.getIsMultipartKey()) {
+ commitUploadPartInfo =
+ omClient.commitMultipartUploadPart(keyArgs, openID);
+ } else {
+ omClient.commitKey(keyArgs, openID);
+ }
+ } else {
+ LOG.warn("Closing KeyDataStreamOutput, but key args is null");
+ }
+ }
+
+ public BlockDataStreamOutputEntry getCurrentStreamEntry() {
+ if (streamEntries.isEmpty() || streamEntries.size() <= currentStreamIndex) {
+ return null;
+ } else {
+ return streamEntries.get(currentStreamIndex);
+ }
+ }
+
+ BlockDataStreamOutputEntry allocateBlockIfNeeded() throws IOException {
+ BlockDataStreamOutputEntry streamEntry = getCurrentStreamEntry();
+ if (streamEntry != null && streamEntry.isClosed()) {
+ // a stream entry gets closed either by :
+ // a. If the stream gets full
+ // b. it has encountered an exception
+ currentStreamIndex++;
+ }
+ if (streamEntries.size() <= currentStreamIndex) {
+ Preconditions.checkNotNull(omClient);
+ // allocate a new block, if a exception happens, log an error and
+ // throw exception to the caller directly, and the write fails.
+ allocateNewBlock();
+ }
+ // in theory, this condition should never violate due the check above
+ // still do a sanity check.
+ Preconditions.checkArgument(currentStreamIndex < streamEntries.size());
+ return streamEntries.get(currentStreamIndex);
+ }
+
+ long computeBufferData() {
+ return bufferPool.computeBufferData();
+ }
+
+ void cleanup() {
+ if (excludeList != null) {
+ excludeList.clear();
+ }
+ if (bufferPool != null) {
+ bufferPool.clearBufferPool();
+ }
+
+ if (streamEntries != null) {
+ streamEntries.clear();
+ }
+ }
+
+ public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() {
+ return commitUploadPartInfo;
+ }
+
+ public ExcludeList getExcludeList() {
+ return excludeList;
+ }
+
+ boolean isEmpty() {
+ return streamEntries.isEmpty();
+ }
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
new file mode 100644
index 0000000000..a9be11667c
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client.io;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import io.netty.buffer.ByteBuf;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
+import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.ratis.protocol.exceptions.AlreadyClosedException;
+import org.apache.ratis.protocol.exceptions.RaftRetryFailureException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Maintaining a list of BlockInputStream. Write based on offset.
+ *
+ * Note that this may write to multiple containers in one write call. In case
+ * that first container succeeded but later ones failed, the succeeded writes
+ * are not rolled back.
+ *
+ * TODO : currently not support multi-thread access.
+ */
+public class KeyDataStreamOutput implements ByteBufStreamOutput {
+
+ private OzoneClientConfig config;
+
+ /**
+ * Defines stream action while calling handleFlushOrClose.
+ */
+ enum StreamAction {
+ FLUSH, CLOSE, FULL
+ }
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(KeyDataStreamOutput.class);
+
+ private boolean closed;
+ private FileEncryptionInfo feInfo;
+ private final Map<Class<? extends Throwable>, RetryPolicy> retryPolicyMap;
+ private int retryCount;
+ // how much of data is actually written yet to underlying stream
+ private long offset;
+ // how much data has been ingested into the stream
+ private long writeOffset;
+ // whether an exception is encountered while write and whole write could
+ // not succeed
+ private boolean isException;
+ private final BlockDataStreamOutputEntryPool blockDataStreamOutputEntryPool;
+
+ private long clientID;
+
+ /**
+ * A constructor for testing purpose only.
+ */
+ @VisibleForTesting
+ public KeyDataStreamOutput() {
+ closed = false;
+ this.retryPolicyMap = HddsClientUtils.getExceptionList()
+ .stream()
+ .collect(Collectors.toMap(Function.identity(),
+ e -> RetryPolicies.TRY_ONCE_THEN_FAIL));
+ retryCount = 0;
+ offset = 0;
+ blockDataStreamOutputEntryPool = new BlockDataStreamOutputEntryPool();
+ }
+
+ @VisibleForTesting
+ public List<BlockDataStreamOutputEntry> getStreamEntries() {
+ return blockDataStreamOutputEntryPool.getStreamEntries();
+ }
+
+ @VisibleForTesting
+ public XceiverClientFactory getXceiverClientFactory() {
+ return blockDataStreamOutputEntryPool.getXceiverClientFactory();
+ }
+
+ @VisibleForTesting
+ public List<OmKeyLocationInfo> getLocationInfoList() {
+ return blockDataStreamOutputEntryPool.getLocationInfoList();
+ }
+
+ @VisibleForTesting
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ @VisibleForTesting
+ public long getClientID() {
+ return clientID;
+ }
+
+ @SuppressWarnings({"parameternumber", "squid:S00107"})
+ public KeyDataStreamOutput(
+ OzoneClientConfig config,
+ OpenKeySession handler,
+ XceiverClientFactory xceiverClientManager,
+ OzoneManagerProtocol omClient, int chunkSize,
+ String requestId, ReplicationConfig replicationConfig,
+ String uploadID, int partNumber, boolean isMultipart,
+ boolean unsafeByteBufferConversion
+ ) {
+ this.config = config;
+ OmKeyInfo info = handler.getKeyInfo();
+ blockDataStreamOutputEntryPool =
+ new BlockDataStreamOutputEntryPool(
+ config,
+ omClient,
+ requestId, replicationConfig,
+ uploadID, partNumber,
+ isMultipart, info,
+ unsafeByteBufferConversion,
+ xceiverClientManager,
+ handler.getId());
+
+ // Retrieve the file encryption key info, null if file is not in
+ // encrypted bucket.
+ this.feInfo = info.getFileEncryptionInfo();
+ this.retryPolicyMap = HddsClientUtils.getRetryPolicyByException(
+ config.getMaxRetryCount(), config.getRetryInterval());
+ this.retryCount = 0;
+ this.isException = false;
+ this.writeOffset = 0;
+ this.clientID = handler.getId();
+ }
+
+ /**
+ * When a key is opened, it is possible that there are some blocks already
+ * allocated to it for this open session. In this case, to make use of these
+ * blocks, we need to add these blocks to stream entries. But, a key's version
+ * also includes blocks from previous versions, we need to avoid adding these
+ * old blocks to stream entries, because these old blocks should not be picked
+ * for write. To do this, the following method checks that, only those
+ * blocks created in this particular open version are added to stream entries.
+ *
+ * @param version the set of blocks that are pre-allocated.
+ * @param openVersion the version corresponding to the pre-allocation.
+ * @throws IOException
+ */
+ public void addPreallocateBlocks(OmKeyLocationInfoGroup version,
+ long openVersion) throws IOException {
+ blockDataStreamOutputEntryPool.addPreallocateBlocks(version, openVersion);
+ }
+
+ @Override
+ public void write(ByteBuf b) throws IOException {
+ checkNotClosed();
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ final int len = b.readableBytes();
+ handleWrite(b, b.readerIndex(), len, false);
+ writeOffset += len;
+ }
+
+ private void handleWrite(ByteBuf b, int off, long len, boolean retry)
+ throws IOException {
+ while (len > 0) {
+ try {
+ BlockDataStreamOutputEntry current =
+ blockDataStreamOutputEntryPool.allocateBlockIfNeeded();
+ // length(len) will be in int range if the call is happening through
+ // write API of blockDataStreamOutput. Length can be in long range
+ // if it comes via Exception path.
+ int expectedWriteLen = Math.min((int) len,
+ (int) current.getRemaining());
+ long currentPos = current.getWrittenDataLength();
+ // writeLen will be updated based on whether the write was succeeded
+ // or if it sees an exception, how much the actual write was
+ // acknowledged.
+ int writtenLength =
+ writeToDataStreamOutput(current, retry, len, b,
+ expectedWriteLen, off, currentPos);
+ if (current.getRemaining() <= 0) {
+ // since the current block is already written close the stream.
+ handleFlushOrClose(StreamAction.FULL);
+ }
+ len -= writtenLength;
+ off += writtenLength;
+ } catch (Exception e) {
+ markStreamClosed();
+ throw new IOException(e);
+ }
+ }
+ }
+
+ private int writeToDataStreamOutput(BlockDataStreamOutputEntry current,
+ boolean retry, long len, ByteBuf b, int writeLen, int off,
+ long currentPos) throws IOException {
+ try {
+ if (retry) {
+ current.writeOnRetry(len);
+ } else {
+ current.write(b, off, writeLen);
+ offset += writeLen;
+ }
+ } catch (IOException ioe) {
+ // for the current iteration, totalDataWritten - currentPos gives the
+ // amount of data already written to the buffer
+
+ // In the retryPath, the total data to be written will always be equal
+ // to or less than the max length of the buffer allocated.
+ // The len specified here is the combined sum of the data length of
+ // the buffers
+ Preconditions.checkState(!retry || len <= config
+ .getStreamBufferMaxSize());
+ int dataWritten = (int) (current.getWrittenDataLength() - currentPos);
+ writeLen = retry ? (int) len : dataWritten;
+ // In retry path, the data written is already accounted in offset.
+ if (!retry) {
+ offset += writeLen;
+ }
+ LOG.debug("writeLen {}, total len {}", writeLen, len);
+ handleException(current, ioe);
+ }
+ return writeLen;
+ }
+
+ /**
+ * It performs following actions :
+ * a. Updates the committed length at datanode for the current stream in
+ * datanode.
+ * b. Reads the data from the underlying buffer and writes it the next stream.
+ *
+ * @param streamEntry StreamEntry
+ * @param exception actual exception that occurred
+ * @throws IOException Throws IOException if Write fails
+ */
+ private void handleException(BlockDataStreamOutputEntry streamEntry,
+ IOException exception) throws IOException {
+ Throwable t = HddsClientUtils.checkForException(exception);
+ Preconditions.checkNotNull(t);
+ boolean retryFailure = checkForRetryFailure(t);
+ boolean containerExclusionException = false;
+ if (!retryFailure) {
+ containerExclusionException = checkIfContainerToExclude(t);
+ }
+ Pipeline pipeline = streamEntry.getPipeline();
+ PipelineID pipelineId = pipeline.getId();
+ long totalSuccessfulFlushedData = streamEntry.getTotalAckDataLength();
+ //set the correct length for the current stream
+ streamEntry.setCurrentPosition(totalSuccessfulFlushedData);
+ long bufferedDataLen = blockDataStreamOutputEntryPool.computeBufferData();
+ if (containerExclusionException) {
+ LOG.debug(
+ "Encountered exception {}. The last committed block length is {}, "
+ + "uncommitted data length is {} retry count {}", exception,
+ totalSuccessfulFlushedData, bufferedDataLen, retryCount);
+ } else {
+ LOG.warn(
+ "Encountered exception {} on the pipeline {}. "
+ + "The last committed block length is {}, "
+ + "uncommitted data length is {} retry count {}", exception,
+ pipeline, totalSuccessfulFlushedData, bufferedDataLen, retryCount);
+ }
+ Preconditions.checkArgument(
+ bufferedDataLen <= config.getStreamBufferMaxSize());
+ Preconditions.checkArgument(
+ offset - blockDataStreamOutputEntryPool.getKeyLength() ==
+ bufferedDataLen);
+ long containerId = streamEntry.getBlockID().getContainerID();
+ Collection<DatanodeDetails> failedServers = streamEntry.getFailedServers();
+ Preconditions.checkNotNull(failedServers);
+ ExcludeList excludeList = blockDataStreamOutputEntryPool.getExcludeList();
+ if (!failedServers.isEmpty()) {
+ excludeList.addDatanodes(failedServers);
+ }
+
+ // if the container needs to be excluded , add the container to the
+ // exclusion list , otherwise add the pipeline to the exclusion list
+ if (containerExclusionException) {
+ excludeList.addConatinerId(ContainerID.valueOf(containerId));
+ } else {
+ excludeList.addPipeline(pipelineId);
+ }
+ // just clean up the current stream.
+ streamEntry.cleanup(retryFailure);
+
+ // discard all subsequent blocks the containers and pipelines which
+ // are in the exclude list so that, the very next retry should never
+ // write data on the closed container/pipeline
+ if (containerExclusionException) {
+ // discard subsequent pre allocated blocks from the streamEntries list
+ // from the closed container
+ blockDataStreamOutputEntryPool
+ .discardPreallocatedBlocks(streamEntry.getBlockID().getContainerID(),
+ null);
+ } else {
+ // In case there is timeoutException or Watch for commit happening over
+ // majority or the client connection failure to the leader in the
+ // pipeline, just discard all the pre allocated blocks on this pipeline.
+ // Next block allocation will happen with excluding this specific pipeline
+ // This will ensure if 2 way commit happens , it cannot span over multiple
+ // blocks
+ blockDataStreamOutputEntryPool
+ .discardPreallocatedBlocks(-1, pipelineId);
+ }
+ if (bufferedDataLen > 0) {
+ // If the data is still cached in the underlying stream, we need to
+ // allocate new block and write this data in the datanode.
+ handleRetry(exception, bufferedDataLen);
+ // reset the retryCount after handling the exception
+ retryCount = 0;
+ }
+ }
+
+ private void markStreamClosed() {
+ blockDataStreamOutputEntryPool.cleanup();
+ closed = true;
+ }
+
+ private void handleRetry(IOException exception, long len) throws IOException {
+ RetryPolicy retryPolicy = retryPolicyMap
+ .get(HddsClientUtils.checkForException(exception).getClass());
+ if (retryPolicy == null) {
+ retryPolicy = retryPolicyMap.get(Exception.class);
+ }
+ RetryPolicy.RetryAction action = null;
+ try {
+ action = retryPolicy.shouldRetry(exception, retryCount, 0, true);
+ } catch (Exception e) {
+ setExceptionAndThrow(new IOException(e));
+ }
+ if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
+ String msg = "";
+ if (action.reason != null) {
+ msg = "Retry request failed. " + action.reason;
+ LOG.error(msg, exception);
+ }
+ setExceptionAndThrow(new IOException(msg, exception));
+ }
+
+ // Throw the exception if the thread is interrupted
+ if (Thread.currentThread().isInterrupted()) {
+ LOG.warn("Interrupted while trying for retry");
+ setExceptionAndThrow(exception);
+ }
+ Preconditions.checkArgument(
+ action.action == RetryPolicy.RetryAction.RetryDecision.RETRY);
+ if (action.delayMillis > 0) {
+ try {
+ Thread.sleep(action.delayMillis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ IOException ioe = (IOException) new InterruptedIOException(
+ "Interrupted: action=" + action + ", retry policy=" + retryPolicy)
+ .initCause(e);
+ setExceptionAndThrow(ioe);
+ }
+ }
+ retryCount++;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Retrying Write request. Already tried {} time(s); " +
+ "retry policy is {} ", retryCount, retryPolicy);
+ }
+ handleWrite(null, 0, len, true);
+ }
+
+ private void setExceptionAndThrow(IOException ioe) throws IOException {
+ isException = true;
+ throw ioe;
+ }
+
+ /**
+ * Checks if the provided exception signifies retry failure in ratis client.
+ * In case of retry failure, ratis client throws RaftRetryFailureException
+ * and all succeeding operations are failed with AlreadyClosedException.
+ */
+ private boolean checkForRetryFailure(Throwable t) {
+ return t instanceof RaftRetryFailureException
+ || t instanceof AlreadyClosedException;
+ }
+
+ // Every container specific exception from datatnode will be seen as
+ // StorageContainerException
+ private boolean checkIfContainerToExclude(Throwable t) {
+ return t instanceof StorageContainerException;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ checkNotClosed();
+ handleFlushOrClose(StreamAction.FLUSH);
+ }
+
+ /**
+ * Close or Flush the latest outputStream depending upon the action.
+ * This function gets called when while write is going on, the current stream
+ * gets full or explicit flush or close request is made by client. when the
+ * stream gets full and we try to close the stream , we might end up hitting
+ * an exception in the exception handling path, we write the data residing in
+ * in the buffer pool to a new Block. In cases, as such, when the data gets
+ * written to new stream , it will be at max half full. In such cases, we
+ * should just write the data and not close the stream as the block won't be
+ * completely full.
+ *
+ * @param op Flag which decides whether to call close or flush on the
+ * outputStream.
+ * @throws IOException In case, flush or close fails with exception.
+ */
+ @SuppressWarnings("squid:S1141")
+ private void handleFlushOrClose(StreamAction op) throws IOException {
+ if (!blockDataStreamOutputEntryPool.isEmpty()) {
+ while (true) {
+ try {
+ BlockDataStreamOutputEntry entry =
+ blockDataStreamOutputEntryPool.getCurrentStreamEntry();
+ if (entry != null) {
+ try {
+ handleStreamAction(entry, op);
+ } catch (IOException ioe) {
+ handleException(entry, ioe);
+ continue;
+ }
+ }
+ return;
+ } catch (Exception e) {
+ markStreamClosed();
+ throw e;
+ }
+ }
+ }
+ }
+
+ private void handleStreamAction(BlockDataStreamOutputEntry entry,
+ StreamAction op) throws IOException {
+ Collection<DatanodeDetails> failedServers = entry.getFailedServers();
+ // failed servers can be null in case there is no data written in
+ // the stream
+ if (!failedServers.isEmpty()) {
+ blockDataStreamOutputEntryPool.getExcludeList().addDatanodes(
+ failedServers);
+ }
+ switch (op) {
+ case CLOSE:
+ entry.close();
+ break;
+ case FULL:
+ if (entry.getRemaining() == 0) {
+ entry.close();
+ }
+ break;
+ case FLUSH:
+ entry.flush();
+ break;
+ default:
+ throw new IOException("Invalid Operation");
+ }
+ }
+
+ /**
+ * Commit the key to OM, this will add the blocks as the new key blocks.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ try {
+ handleFlushOrClose(StreamAction.CLOSE);
+ if (!isException) {
+ Preconditions.checkArgument(writeOffset == offset);
+ }
+ blockDataStreamOutputEntryPool.commitKey(offset);
+ } finally {
+ blockDataStreamOutputEntryPool.cleanup();
+ }
+ }
+
+ public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() {
+ return blockDataStreamOutputEntryPool.getCommitUploadPartInfo();
+ }
+
+ public FileEncryptionInfo getFileEncryptionInfo() {
+ return feInfo;
+ }
+
+ @VisibleForTesting
+ public ExcludeList getExcludeList() {
+ return blockDataStreamOutputEntryPool.getExcludeList();
+ }
+
+ /**
+ * Builder class of KeyDataStreamOutput.
+ */
+ public static class Builder {
+ private OpenKeySession openHandler;
+ private XceiverClientFactory xceiverManager;
+ private OzoneManagerProtocol omClient;
+ private int chunkSize;
+ private String requestID;
+ private String multipartUploadID;
+ private int multipartNumber;
+ private boolean isMultipartKey;
+ private boolean unsafeByteBufferConversion;
+ private OzoneClientConfig clientConfig;
+ private ReplicationConfig replicationConfig;
+
+ public Builder setMultipartUploadID(String uploadID) {
+ this.multipartUploadID = uploadID;
+ return this;
+ }
+
+ public Builder setMultipartNumber(int partNumber) {
+ this.multipartNumber = partNumber;
+ return this;
+ }
+
+ public Builder setHandler(OpenKeySession handler) {
+ this.openHandler = handler;
+ return this;
+ }
+
+ public Builder setXceiverClientManager(XceiverClientFactory manager) {
+ this.xceiverManager = manager;
+ return this;
+ }
+
+ public Builder setOmClient(OzoneManagerProtocol client) {
+ this.omClient = client;
+ return this;
+ }
+
+ public Builder setChunkSize(int size) {
+ this.chunkSize = size;
+ return this;
+ }
+
+ public Builder setRequestID(String id) {
+ this.requestID = id;
+ return this;
+ }
+
+ public Builder setIsMultipartKey(boolean isMultipart) {
+ this.isMultipartKey = isMultipart;
+ return this;
+ }
+
+ public Builder setConfig(OzoneClientConfig config) {
+ this.clientConfig = config;
+ return this;
+ }
+
+ public Builder enableUnsafeByteBufferConversion(boolean enabled) {
+ this.unsafeByteBufferConversion = enabled;
+ return this;
+ }
+
+
+ public Builder setReplicationConfig(ReplicationConfig replConfig) {
+ this.replicationConfig = replConfig;
+ return this;
+ }
+
+ public KeyDataStreamOutput build() {
+ return new KeyDataStreamOutput(
+ clientConfig,
+ openHandler,
+ xceiverManager,
+ omClient,
+ chunkSize,
+ requestID,
+ replicationConfig,
+ multipartUploadID,
+ multipartNumber,
+ isMultipartKey,
+ unsafeByteBufferConversion);
+ }
+
+ }
+
+ /**
+ * Verify that the output stream is open. Non blocking; this gives
+ * the last state of the volatile {@link #closed} field.
+ * @throws IOException if the connection is closed.
+ */
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException(
+ ": " + FSExceptionMessages.STREAM_IS_CLOSED + " Key: "
+ + blockDataStreamOutputEntryPool.getKeyName());
+ }
+ }
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
new file mode 100644
index 0000000000..378b86872e
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.client.io;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
+
+import java.io.IOException;
+
+/**
+ * OzoneDataStreamOutput is used to write data into Ozone.
+ * It uses SCM's {@link KeyDataStreamOutput} for writing the data.
+ */
+public class OzoneDataStreamOutput implements ByteBufStreamOutput {
+
+ private final ByteBufStreamOutput byteBufStreamOutput;
+
+ /**
+ * Constructs OzoneDataStreamOutput with KeyDataStreamOutput.
+ *
+ * @param byteBufStreamOutput
+ */
+ public OzoneDataStreamOutput(ByteBufStreamOutput byteBufStreamOutput) {
+ this.byteBufStreamOutput = byteBufStreamOutput;
+ }
+
+ @Override
+ public void write(ByteBuf b) throws IOException {
+ byteBufStreamOutput.write(b);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ byteBufStreamOutput.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ //commitKey can be done here, if needed.
+ byteBufStreamOutput.close();
+ }
+
+ public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() {
+ if (byteBufStreamOutput instanceof KeyDataStreamOutput) {
+ return ((KeyDataStreamOutput)
+ byteBufStreamOutput).getCommitUploadPartInfo();
+ }
+ // Otherwise return null.
+ return null;
+ }
+
+ public ByteBufStreamOutput getByteBufStreamOutput() {
+ return byteBufStreamOutput;
+ }
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 725eb3a586..de800b113d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -294,6 +295,20 @@ public interface ClientProtocol {
Map<String, String> metadata)
throws IOException;
+ /**
+ * Writes a key in an existing bucket.
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param metadata custom key value metadata
+ * @return {@link OzoneDataStreamOutput}
+ *
+ */
+ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata)
+ throws IOException;
/**
* Reads a key from an existing bucket.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 213868cb6c..4d8ed2dd91 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -89,11 +89,13 @@ import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
+import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
import org.apache.hadoop.ozone.client.io.MultipartCryptoKeyInputStream;
import org.apache.hadoop.ozone.client.io.OzoneCryptoInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
@@ -985,6 +987,48 @@ public class RpcClient implements ClientProtocol {
return createOutputStream(openKey, requestId);
}
+ @Override
+ public OzoneDataStreamOutput createStreamKey(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata)
+ throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ if (checkKeyNameEnabled) {
+ HddsClientUtils.verifyKeyName(keyName);
+ }
+ HddsClientUtils.checkNotNull(keyName, replicationConfig);
+ String requestId = UUID.randomUUID().toString();
+
+ OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(size)
+ .setReplicationConfig(replicationConfig)
+ .addAllMetadata(metadata)
+ .setAcls(getAclList());
+
+ if (Boolean.parseBoolean(metadata.get(OzoneConsts.GDPR_FLAG))) {
+ try{
+ GDPRSymmetricKey gKey = new GDPRSymmetricKey(new SecureRandom());
+ builder.addAllMetadata(gKey.getKeyDetails());
+ } catch (Exception e) {
+ if (e instanceof InvalidKeyException &&
+ e.getMessage().contains("Illegal key size or default parameters")) {
+ LOG.error("Missing Unlimited Strength Policy jars. Please install " +
+ "Java Cryptography Extension (JCE) Unlimited Strength " +
+ "Jurisdiction Policy Files");
+ }
+ throw new IOException(e);
+ }
+ }
+
+ OpenKeySession openKey = ozoneManagerClient.openKey(builder.build());
+ return createDataStreamOutput(openKey, requestId, replicationConfig);
+ }
+
private KeyProvider.KeyVersion getDEK(FileEncryptionInfo feInfo)
throws IOException {
// check crypto protocol version
@@ -1647,6 +1691,24 @@ public class RpcClient implements ClientProtocol {
cryptoInputStreams);
}
}
+ private OzoneDataStreamOutput createDataStreamOutput(OpenKeySession openKey,
+ String requestId, ReplicationConfig replicationConfig)
+ throws IOException {
+ KeyDataStreamOutput keyOutputStream =
+ new KeyDataStreamOutput.Builder()
+ .setHandler(openKey)
+ .setXceiverClientManager(xceiverClientManager)
+ .setOmClient(ozoneManagerClient)
+ .setRequestID(requestId)
+ .setReplicationConfig(replicationConfig)
+ .enableUnsafeByteBufferConversion(unsafeByteBufferConversion)
+ .setConfig(clientConfig)
+ .build();
+ keyOutputStream
+ .addPreallocateBlocks(openKey.getKeyInfo().getLatestVersionLocations(),
+ openKey.getOpenVersion());
+ return new OzoneDataStreamOutput(keyOutputStream);
+ }
private OzoneOutputStream createOutputStream(OpenKeySession openKey,
String requestId) throws IOException {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
new file mode 100644
index 0000000000..4d52d89490
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.client.rpc;
+
+import io.netty.buffer.Unpooled;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+
+/**
+ * Tests BlockDataStreamOutput class.
+ */
+public class TestBlockDataStreamOutput {
+
+ /**
+ * Set a timeout for each test.
+ */
+ @Rule
+ public Timeout timeout = Timeout.seconds(300);
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ private static OzoneClient client;
+ private static ObjectStore objectStore;
+ private static int chunkSize;
+ private static int flushSize;
+ private static int maxFlushSize;
+ private static int blockSize;
+ private static String volumeName;
+ private static String bucketName;
+ private static String keyString;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ chunkSize = 100;
+ flushSize = 2 * chunkSize;
+ maxFlushSize = 2 * flushSize;
+ blockSize = 2 * maxFlushSize;
+
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setChecksumType(ChecksumType.NONE);
+ clientConfig.setStreamBufferFlushDelay(false);
+ conf.setFromObject(clientConfig);
+
+ conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
+ conf.setQuietMode(false);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
+
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
+ .setBlockSize(blockSize)
+ .setChunkSize(chunkSize)
+ .setStreamBufferFlushSize(flushSize)
+ .setStreamBufferMaxSize(maxFlushSize)
+ .setStreamBufferSizeUnit(StorageUnit.BYTES)
+ .build();
+ cluster.waitForClusterToBeReady();
+ //the easiest way to create an open container is creating a key
+ client = OzoneClientFactory.getRpcClient(conf);
+ objectStore = client.getObjectStore();
+ keyString = UUID.randomUUID().toString();
+ volumeName = "testblockoutputstream";
+ bucketName = volumeName;
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+ }
+
+ private String getKeyName() {
+ return UUID.randomUUID().toString();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testMultiChunkWrite() throws Exception {
+ // write data less than 1 chunk size use streaming.
+ String keyName1 = getKeyName();
+ OzoneDataStreamOutput key1 = createKey(
+ keyName1, ReplicationType.RATIS, 0);
+ int dataLength1 = chunkSize/2;
+ byte[] data1 =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength1)
+ .getBytes(UTF_8);
+ key1.write(Unpooled.copiedBuffer(data1));
+ // now close the stream, It will update the key length.
+ key1.close();
+ validateData(keyName1, data1);
+
+ // write data more than 1 chunk size use streaming.
+ String keyName2 = getKeyName();
+ OzoneDataStreamOutput key2 = createKey(
+ keyName2, ReplicationType.RATIS, 0);
+ int dataLength2 = chunkSize + 50;
+ byte[] data2 =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength2)
+ .getBytes(UTF_8);
+ key2.write(Unpooled.copiedBuffer(data2));
+ // now close the stream, It will update the key length.
+ key2.close();
+ validateData(keyName2, data2);
+
+ // write data more than 1 block size use streaming.
+ String keyName3 = getKeyName();
+ OzoneDataStreamOutput key3 = createKey(
+ keyName3, ReplicationType.RATIS, 0);
+ int dataLength3 = blockSize + 50;
+ byte[] data3 =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength3)
+ .getBytes(UTF_8);
+ key3.write(Unpooled.copiedBuffer(data3));
+ // now close the stream, It will update the key length.
+ key3.close();
+ validateData(keyName3, data3);
+ }
+
+ private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
+ long size) throws Exception {
+ return TestHelper.createStreamKey(
+ keyName, type, size, objectStore, volumeName, bucketName);
+ }
+ private void validateData(String keyName, byte[] data) throws Exception {
+ TestHelper
+ .validateData(keyName, data, objectStore, volumeName, bucketName);
+ }
+
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
index dae6e383f8..cf3a51241e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -134,8 +135,23 @@ public final class TestHelper {
}
org.apache.hadoop.hdds.client.ReplicationFactor factor =
org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
+ ReplicationConfig config =
+ ReplicationConfig.fromTypeAndFactor(type, factor);
return objectStore.getVolume(volumeName).getBucket(bucketName)
- .createKey(keyName, size, type, factor, new HashMap<>());
+ .createKey(keyName, size, config, new HashMap<>());
+ }
+
+ public static OzoneDataStreamOutput createStreamKey(String keyName,
+ ReplicationType type, long size, ObjectStore objectStore,
+ String volumeName, String bucketName) throws Exception {
+ org.apache.hadoop.hdds.client.ReplicationFactor factor =
+ type == ReplicationType.STAND_ALONE ?
+ org.apache.hadoop.hdds.client.ReplicationFactor.ONE :
+ org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
+ ReplicationConfig config =
+ ReplicationConfig.fromTypeAndFactor(type, factor);
+ return objectStore.getVolume(volumeName).getBucket(bucketName)
+ .createStreamKey(keyName, size, config, new HashMap<>());
}
public static OzoneOutputStream createKey(String keyName,
@@ -143,8 +159,10 @@ public final class TestHelper {
org.apache.hadoop.hdds.client.ReplicationFactor factor, long size,
ObjectStore objectStore, String volumeName, String bucketName)
throws Exception {
+ ReplicationConfig config =
+ ReplicationConfig.fromTypeAndFactor(type, factor);
return objectStore.getVolume(volumeName).getBucket(bucketName)
- .createKey(keyName, size, type, factor, new HashMap<>());
+ .createKey(keyName, size, config, new HashMap<>());
}
public static OzoneOutputStream createKey(String keyName,
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index 7d7885d168..e4b842eaca 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -23,9 +23,14 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.Map;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.io.IOUtils;
@@ -34,6 +39,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.shell.OzoneAddress;
import org.apache.commons.codec.digest.DigestUtils;
@@ -89,10 +95,36 @@ public class PutKeyHandler extends KeyHandler {
int chunkSize = (int) getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
- try (InputStream input = new FileInputStream(dataFile);
- OutputStream output = bucket.createKey(keyName, dataFile.length(),
- replicationConfig, keyMetadata)) {
- IOUtils.copyBytes(input, output, chunkSize);
+
+ if (dataFile.length() <= chunkSize) {
+ if (isVerbose()) {
+ out().println("API: async");
+ }
+ try (InputStream input = new FileInputStream(dataFile);
+ OutputStream output = bucket.createKey(keyName, dataFile.length(),
+ replicationConfig, keyMetadata)) {
+ IOUtils.copyBytes(input, output, chunkSize);
+ }
+ } else {
+ if (isVerbose()) {
+ out().println("API: streaming");
+ }
+ try (RandomAccessFile raf = new RandomAccessFile(dataFile, "r");
+ OzoneDataStreamOutput out = bucket.createStreamKey(keyName,
+ dataFile.length(), replicationConfig, keyMetadata)) {
+ FileChannel ch = raf.getChannel();
+ long len = raf.length();
+ long off = 0;
+ while (len > 0) {
+ long writeLen = Math.min(len, chunkSize);
+ ByteBuffer segment =
+ ch.map(FileChannel.MapMode.READ_ONLY, off, writeLen);
+ ByteBuf buf = Unpooled.wrappedBuffer(segment);
+ out.write(buf);
+ off += writeLen;
+ len -= writeLen;
+ }
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 34/35: HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at the end of the stream. (#3229)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 246b0a6f2d9329bcf8e1bf647b12a80d62ffc240
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Mon Mar 28 20:43:38 2022 +0800
HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at the end of the stream. (#3229)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 48 +++-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 26 ++
.../hdds/scm/storage/ContainerProtocolCalls.java | 14 +-
.../server/ratis/ContainerStateMachine.java | 27 +-
.../keyvalue/impl/KeyValueStreamDataChannel.java | 235 ++++++++++++++++
.../keyvalue/impl/StreamDataChannelBase.java | 3 +-
.../impl/TestKeyValueStreamDataChannel.java | 313 +++++++++++++++++++++
.../client/rpc/TestBlockDataStreamOutput.java | 16 +-
pom.xml | 2 +-
9 files changed, 658 insertions(+), 26 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index d5b9dd9d81..d19f2aea13 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
@@ -83,6 +84,9 @@ import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putBlock
public class BlockDataStreamOutput implements ByteBufferStreamOutput {
public static final Logger LOG =
LoggerFactory.getLogger(BlockDataStreamOutput.class);
+
+ public static final int PUT_BLOCK_REQUEST_LENGTH_MAX = 1 << 20; // 1MB
+
public static final String EXCEPTION_MSG =
"Unexpected Storage Container Exception: ";
private static final CompletableFuture[] EMPTY_FUTURE_ARRAY = {};
@@ -406,12 +410,26 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
byteBufferList = null;
}
waitFuturesComplete();
+ final BlockData blockData = containerBlockData.build();
if (close) {
- dataStreamCloseReply = out.closeAsync();
+ final ContainerCommandRequestProto putBlockRequest
+ = ContainerProtocolCalls.getPutBlockRequest(
+ xceiverClient.getPipeline(), blockData, true, token);
+ dataStreamCloseReply = executePutBlockClose(putBlockRequest,
+ PUT_BLOCK_REQUEST_LENGTH_MAX, out);
+ dataStreamCloseReply.whenComplete((reply, e) -> {
+ if (e != null || reply == null || !reply.isSuccess()) {
+ LOG.warn("Failed executePutBlockClose, reply=" + reply, e);
+ try {
+ executePutBlock(true, false);
+ } catch (IOException ex) {
+ throw new CompletionException(ex);
+ }
+ }
+ });
}
try {
- BlockData blockData = containerBlockData.build();
XceiverClientReply asyncReply =
putBlockAsync(xceiverClient, blockData, close, token);
final CompletableFuture<ContainerCommandResponseProto> flushFuture
@@ -459,6 +477,30 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
}
+ public static CompletableFuture<DataStreamReply> executePutBlockClose(
+ ContainerCommandRequestProto putBlockRequest, int max,
+ DataStreamOutput out) {
+ final ByteBuffer putBlock = ContainerCommandRequestMessage.toMessage(
+ putBlockRequest, null).getContent().asReadOnlyByteBuffer();
+ final ByteBuffer protoLength = getProtoLength(putBlock, max);
+ RatisHelper.debug(putBlock, "putBlock", LOG);
+ out.writeAsync(putBlock);
+ RatisHelper.debug(protoLength, "protoLength", LOG);
+ return out.writeAsync(protoLength, StandardWriteOption.CLOSE);
+ }
+
+ public static ByteBuffer getProtoLength(ByteBuffer putBlock, int max) {
+ final int protoLength = putBlock.remaining();
+ Preconditions.checkState(protoLength <= max,
+ "protoLength== %s > max = %s", protoLength, max);
+ final ByteBuffer buffer = ByteBuffer.allocate(4);
+ buffer.putInt(protoLength);
+ buffer.flip();
+ LOG.debug("protoLength = {}", protoLength);
+ Preconditions.checkState(buffer.remaining() == 4);
+ return buffer.asReadOnlyBuffer();
+ }
+
@Override
public void flush() throws IOException {
if (xceiverClientFactory != null && xceiverClient != null
@@ -547,7 +589,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
- private void setIoException(Exception e) {
+ private void setIoException(Throwable e) {
IOException ioe = getIoException();
if (ioe == null) {
IOException exception = new IOException(EXCEPTION_MSG + e.toString(), e);
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index b381d7a3a1..902b8e2c0f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.ratis;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collection;
@@ -58,6 +59,7 @@ import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -436,4 +438,28 @@ public final class RatisHelper {
throw new RuntimeException(e);
}
}
+
+ public static void debug(ByteBuffer buffer, String name, Logger log) {
+ if (!log.isDebugEnabled()) {
+ return;
+ }
+ buffer = buffer.duplicate();
+ final StringBuilder builder = new StringBuilder();
+ for (int i = 1; buffer.remaining() > 0; i++) {
+ builder.append(buffer.get()).append(i % 20 == 0 ? "\n " : ", ");
+ }
+ log.debug("{}: {}\n {}", name, buffer, builder);
+ }
+
+ public static void debug(ByteBuf buf, String name, Logger log) {
+ if (!log.isDebugEnabled()) {
+ return;
+ }
+ buf = buf.duplicate();
+ final StringBuilder builder = new StringBuilder();
+ for (int i = 1; buf.readableBytes() > 0; i++) {
+ builder.append(buf.readByte()).append(i % 20 == 0 ? "\n " : ", ");
+ }
+ log.debug("{}: {}\n {}", name, buf, builder);
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 259787070e..022c26e4ca 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.BlockNotCommittedException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
@@ -186,11 +187,19 @@ public final class ContainerProtocolCalls {
XceiverClientSpi xceiverClient, BlockData containerBlockData, boolean eof,
Token<? extends TokenIdentifier> token)
throws IOException, InterruptedException, ExecutionException {
+ final ContainerCommandRequestProto request = getPutBlockRequest(
+ xceiverClient.getPipeline(), containerBlockData, eof, token);
+ return xceiverClient.sendCommandAsync(request);
+ }
+
+ public static ContainerCommandRequestProto getPutBlockRequest(
+ Pipeline pipeline, BlockData containerBlockData, boolean eof,
+ Token<? extends TokenIdentifier> token) throws IOException {
PutBlockRequestProto.Builder createBlockRequest =
PutBlockRequestProto.newBuilder()
.setBlockData(containerBlockData)
.setEof(eof);
- String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
+ final String id = pipeline.getFirstNode().getUuidString();
ContainerCommandRequestProto.Builder builder =
ContainerCommandRequestProto.newBuilder().setCmdType(Type.PutBlock)
.setContainerID(containerBlockData.getBlockID().getContainerID())
@@ -199,8 +208,7 @@ public final class ContainerProtocolCalls {
if (token != null) {
builder.setEncodedToken(token.encodeToUrlString());
}
- ContainerCommandRequestProto request = builder.build();
- return xceiverClient.sendCommandAsync(request);
+ return builder.build();
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 138cddd486..3edb29b94a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -427,6 +427,20 @@ public class ContainerStateMachine extends BaseStateMachine {
return dispatchCommand(requestProto, context);
}
+ private CompletableFuture<ContainerCommandResponseProto> runCommandAsync(
+ ContainerCommandRequestProto requestProto, LogEntryProto entry) {
+ return CompletableFuture.supplyAsync(() -> {
+ final DispatcherContext context = new DispatcherContext.Builder()
+ .setTerm(entry.getTerm())
+ .setLogIndex(entry.getIndex())
+ .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA)
+ .setContainer2BCSIDMap(container2BCSIDMap)
+ .build();
+
+ return runCommand(requestProto, context);
+ }, executor);
+ }
+
private CompletableFuture<Message> handleWriteChunk(
ContainerCommandRequestProto requestProto, long entryIndex, long term,
long startTime) {
@@ -560,19 +574,16 @@ public class ContainerStateMachine extends BaseStateMachine {
"DataStream: " + stream + " is not closed properly"));
}
- final CompletableFuture<ContainerCommandResponseProto> f;
+ final ContainerCommandRequestProto request;
if (dataChannel instanceof KeyValueStreamDataChannel) {
- f = CompletableFuture.completedFuture(null);
+ request = ((KeyValueStreamDataChannel) dataChannel).getPutBlockRequest();
} else {
return JavaUtils.completeExceptionally(new IllegalStateException(
"Unexpected DataChannel " + dataChannel.getClass()));
}
- return f.whenComplete((res, e) -> {
- if (LOG.isDebugEnabled()) {
- LOG.debug("PutBlock {} Term: {} Index: {}",
- res.getResult(), entry.getTerm(), entry.getIndex());
- }
- });
+ return runCommandAsync(request, entry).whenComplete(
+ (res, e) -> LOG.debug("link {}, entry: {}, request: {}",
+ res.getResult(), entry, request));
}
private ExecutorService getChunkExecutor(WriteChunkRequestProto req) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
index 66723031f0..99dc40f5d0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
@@ -18,17 +18,131 @@
package org.apache.hadoop.ozone.container.keyvalue.impl;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf;
+import org.apache.ratis.thirdparty.io.netty.buffer.Unpooled;
+import org.apache.ratis.util.ReferenceCountedObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
/**
* This class is used to get the DataChannel for streaming.
*/
public class KeyValueStreamDataChannel extends StreamDataChannelBase {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(KeyValueStreamDataChannel.class);
+
+ /**
+ * Keep the last {@link Buffers#max} bytes in the buffer
+ * in order to create putBlockRequest
+ * at {@link #closeBuffers(Buffers, WriteMethod)}}.
+ */
+ static class Buffers {
+ private final Deque<ReferenceCountedObject<ByteBuffer>> deque
+ = new LinkedList<>();
+ private final int max;
+ private int length;
+
+ Buffers(int max) {
+ this.max = max;
+ }
+
+ private boolean isExtra(int n) {
+ return length - n >= max;
+ }
+
+ private boolean hasExtraBuffer() {
+ return Optional.ofNullable(deque.peek())
+ .map(ReferenceCountedObject::get)
+ .filter(b -> isExtra(b.remaining()))
+ .isPresent();
+ }
+
+ /**
+ * @return extra buffers which are safe to be written.
+ */
+ Iterable<ReferenceCountedObject<ByteBuffer>> offer(
+ ReferenceCountedObject<ByteBuffer> ref) {
+ final ByteBuffer buffer = ref.retain();
+ LOG.debug("offer {}", buffer);
+ final boolean offered = deque.offer(ref);
+ Preconditions.checkState(offered, "Failed to offer");
+ length += buffer.remaining();
+
+ return () -> new Iterator<ReferenceCountedObject<ByteBuffer>>() {
+ @Override
+ public boolean hasNext() {
+ return hasExtraBuffer();
+ }
+
+ @Override
+ public ReferenceCountedObject<ByteBuffer> next() {
+ final ReferenceCountedObject<ByteBuffer> polled = poll();
+ length -= polled.get().remaining();
+ Preconditions.checkState(length >= max);
+ return polled;
+ }
+ };
+ }
+
+ ReferenceCountedObject<ByteBuffer> poll() {
+ final ReferenceCountedObject<ByteBuffer> polled
+ = Objects.requireNonNull(deque.poll());
+ RatisHelper.debug(polled.get(), "polled", LOG);
+ return polled;
+ }
+
+ ReferenceCountedObject<ByteBuf> pollAll() {
+ Preconditions.checkState(!deque.isEmpty(), "The deque is empty");
+ final ByteBuffer[] array = new ByteBuffer[deque.size()];
+ final List<ReferenceCountedObject<ByteBuffer>> refs
+ = new ArrayList<>(deque.size());
+ for (int i = 0; i < array.length; i++) {
+ final ReferenceCountedObject<ByteBuffer> ref = poll();
+ refs.add(ref);
+ array[i] = ref.get();
+ }
+ final ByteBuf buf = Unpooled.wrappedBuffer(array).asReadOnly();
+ return ReferenceCountedObject.wrap(buf, () -> {
+ }, () -> {
+ buf.release();
+ refs.forEach(ReferenceCountedObject::release);
+ });
+ }
+ }
+
+ interface WriteMethod {
+ int applyAsInt(ByteBuffer src) throws IOException;
+ }
+
+ private final Buffers buffers = new Buffers(
+ BlockDataStreamOutput.PUT_BLOCK_REQUEST_LENGTH_MAX);
+ private final AtomicReference<ContainerCommandRequestProto> putBlockRequest
+ = new AtomicReference<>();
+ private final AtomicBoolean closed = new AtomicBoolean();
+
KeyValueStreamDataChannel(File file, ContainerData containerData,
ContainerMetrics metrics)
throws StorageContainerException {
@@ -39,4 +153,125 @@ public class KeyValueStreamDataChannel extends StreamDataChannelBase {
ContainerProtos.Type getType() {
return ContainerProtos.Type.StreamWrite;
}
+
+ @Override
+ public int write(ReferenceCountedObject<ByteBuffer> referenceCounted)
+ throws IOException {
+ assertOpen();
+ return writeBuffers(referenceCounted, buffers, super::writeFileChannel);
+ }
+
+ static int writeBuffers(ReferenceCountedObject<ByteBuffer> src,
+ Buffers buffers, WriteMethod writeMethod)
+ throws IOException {
+ for (ReferenceCountedObject<ByteBuffer> b : buffers.offer(src)) {
+ try {
+ writeFully(b.get(), writeMethod);
+ } finally {
+ b.release();
+ }
+ }
+ return src.get().remaining();
+ }
+
+ private static void writeFully(ByteBuffer b, WriteMethod writeMethod)
+ throws IOException {
+ for (; b.remaining() > 0;) {
+ final int written = writeMethod.applyAsInt(b);
+ if (written <= 0) {
+ throw new IOException("Unable to write");
+ }
+ }
+ }
+
+ public ContainerCommandRequestProto getPutBlockRequest() {
+ return Objects.requireNonNull(putBlockRequest.get(),
+ () -> "putBlockRequest == null, " + this);
+ }
+
+ void assertOpen() throws IOException {
+ if (closed.get()) {
+ throw new IOException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed.compareAndSet(false, true)) {
+ putBlockRequest.set(closeBuffers(buffers, super::writeFileChannel));
+ super.close();
+ }
+ }
+
+ static ContainerCommandRequestProto closeBuffers(
+ Buffers buffers, WriteMethod writeMethod) throws IOException {
+ final ReferenceCountedObject<ByteBuf> ref = buffers.pollAll();
+ final ByteBuf buf = ref.retain();
+ final ContainerCommandRequestProto putBlockRequest;
+ try {
+ putBlockRequest = readPutBlockRequest(buf);
+ // write the remaining data
+ writeFully(buf.nioBuffer(), writeMethod);
+ } finally {
+ ref.release();
+ }
+ return putBlockRequest;
+ }
+
+ private static int readProtoLength(ByteBuf b, int lengthIndex) {
+ final int readerIndex = b.readerIndex();
+ LOG.debug("{}, lengthIndex = {}, readerIndex = {}",
+ b, lengthIndex, readerIndex);
+ if (lengthIndex > readerIndex) {
+ b.readerIndex(lengthIndex);
+ } else {
+ Preconditions.checkState(lengthIndex == readerIndex);
+ }
+ RatisHelper.debug(b, "readProtoLength", LOG);
+ return b.nioBuffer().getInt();
+ }
+
+ static ContainerCommandRequestProto readPutBlockRequest(ByteBuf b)
+ throws IOException {
+ // readerIndex protoIndex lengthIndex readerIndex+readableBytes
+ // V V V V
+ // format: |--- data ---|--- proto ---|--- proto length (4 bytes) ---|
+ final int readerIndex = b.readerIndex();
+ final int lengthIndex = readerIndex + b.readableBytes() - 4;
+ final int protoLength = readProtoLength(b.duplicate(), lengthIndex);
+ final int protoIndex = lengthIndex - protoLength;
+
+ final ContainerCommandRequestProto proto;
+ try {
+ proto = readPutBlockRequest(b.slice(protoIndex, protoLength).nioBuffer());
+ } catch (Throwable t) {
+ RatisHelper.debug(b, "catch", LOG);
+ throw new IOException("Failed to readPutBlockRequest from " + b
+ + ": readerIndex=" + readerIndex
+ + ", protoIndex=" + protoIndex
+ + ", protoLength=" + protoLength
+ + ", lengthIndex=" + lengthIndex, t);
+ }
+
+ // set index for reading data
+ b.writerIndex(protoIndex);
+
+ return proto;
+ }
+
+ private static ContainerCommandRequestProto readPutBlockRequest(ByteBuffer b)
+ throws IOException {
+ RatisHelper.debug(b, "readPutBlockRequest", LOG);
+ final ByteString byteString = ByteString.copyFrom(b);
+
+ final ContainerCommandRequestProto request =
+ ContainerCommandRequestMessage.toProto(byteString, null);
+
+ if (!request.hasPutBlock()) {
+ throw new StorageContainerException(
+ "Malformed PutBlock request. trace ID: " + request.getTraceID(),
+ ContainerProtos.Result.MALFORMED_REQUEST);
+ }
+ return request;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
index b31e2ccbf4..9829033248 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
@@ -79,8 +79,7 @@ abstract class StreamDataChannelBase implements StateMachine.DataChannel {
randomAccessFile.close();
}
- @Override
- public int write(ByteBuffer src) throws IOException {
+ final int writeFileChannel(ByteBuffer src) throws IOException {
final int writeBytes = getChannel().write(src);
metrics.incContainerBytesStats(getType(), writeBytes);
containerData.updateWriteStats(writeBytes, false);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestKeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestKeyValueStreamDataChannel.java
new file mode 100644
index 0000000000..d252b1cb1b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestKeyValueStreamDataChannel.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.DatanodeBlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
+import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel.Buffers;
+import org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel.WriteMethod;
+import org.apache.ratis.client.api.DataStreamOutput;
+import org.apache.ratis.io.FilePositionCount;
+import org.apache.ratis.io.StandardWriteOption;
+import org.apache.ratis.io.WriteOption;
+import org.apache.ratis.proto.RaftProtos.CommitInfoProto;
+import org.apache.ratis.proto.RaftProtos.DataStreamPacketHeaderProto;
+import org.apache.ratis.protocol.ClientId;
+import org.apache.ratis.protocol.DataStreamReply;
+import org.apache.ratis.protocol.RaftClientReply;
+import org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf;
+import org.apache.ratis.thirdparty.io.netty.buffer.Unpooled;
+import org.apache.ratis.util.ReferenceCountedObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput.PUT_BLOCK_REQUEST_LENGTH_MAX;
+import static org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput.executePutBlockClose;
+import static org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput.getProtoLength;
+import static org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel.closeBuffers;
+import static org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel.readPutBlockRequest;
+import static org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel.writeBuffers;
+
+/** For testing {@link KeyValueStreamDataChannel}. */
+public class TestKeyValueStreamDataChannel {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestKeyValueStreamDataChannel.class);
+
+ static final ContainerCommandRequestProto PUT_BLOCK_PROTO
+ = ContainerCommandRequestProto.newBuilder()
+ .setCmdType(Type.PutBlock)
+ .setPutBlock(PutBlockRequestProto.newBuilder().setBlockData(
+ BlockData.newBuilder().setBlockID(DatanodeBlockID.newBuilder()
+ .setContainerID(222).setLocalID(333).build()).build()))
+ .setDatanodeUuid("datanodeId")
+ .setContainerID(111L)
+ .build();
+ static final int PUT_BLOCK_PROTO_SIZE = PUT_BLOCK_PROTO.toByteString().size();
+ static {
+ LOG.info("PUT_BLOCK_PROTO_SIZE = {}", PUT_BLOCK_PROTO_SIZE);
+ }
+
+ @Test
+ public void testSerialization() throws Exception {
+ final int max = PUT_BLOCK_REQUEST_LENGTH_MAX;
+ final ByteBuffer putBlockBuf = ContainerCommandRequestMessage.toMessage(
+ PUT_BLOCK_PROTO, null).getContent().asReadOnlyByteBuffer();
+ final ByteBuffer protoLengthBuf = getProtoLength(putBlockBuf, max);
+
+ // random data size
+ final int dataSize = ThreadLocalRandom.current().nextInt(1000) + 100;
+ final byte[] data = new byte[dataSize];
+
+ //serialize
+ final ByteBuf buf = Unpooled.buffer(max);
+ buf.writeBytes(data);
+ buf.writeBytes(putBlockBuf);
+ buf.writeBytes(protoLengthBuf);
+
+ final ContainerCommandRequestProto proto = readPutBlockRequest(buf);
+ Assert.assertEquals(PUT_BLOCK_PROTO, proto);
+ }
+
+ @Test
+ public void testBuffers() throws Exception {
+ final ExecutorService executor = Executors.newFixedThreadPool(32);
+ final List<CompletableFuture<String>> futures = new ArrayList<>();
+
+ final int min = PUT_BLOCK_PROTO_SIZE + 4;
+ final int[] maxValues = {min, 2 * min, 10 * min};
+ final int[] dataSizes = {0, 10, 100, 10_000};
+ for (int max : maxValues) {
+ for (int dataSize : dataSizes) {
+ futures.add(CompletableFuture.supplyAsync(
+ () -> runTestBuffers(dataSize, max), executor));
+ }
+ }
+
+ for (CompletableFuture<String> f : futures) {
+ f.get();
+ }
+ }
+
+ static String runTestBuffers(int dataSize, int max) {
+ final int seed = ThreadLocalRandom.current().nextInt();
+ final String name = String.format("[dataSize=%d,max=%d,seed=%H]",
+ dataSize, max, seed);
+ LOG.info(name);
+ try {
+ runTestBuffers(dataSize, max, seed, name);
+ } catch (Throwable t) {
+ throw new CompletionException("Failed " + name, t);
+ }
+ return name;
+ }
+
+ static void runTestBuffers(int dataSize, int max, int seed, String name)
+ throws Exception {
+ Assert.assertTrue(max >= PUT_BLOCK_PROTO_SIZE);
+
+ // random data
+ final byte[] data = new byte[dataSize];
+ final Random random = new Random(seed);
+ random.nextBytes(data);
+
+ // write output
+ final Buffers buffers = new Buffers(max);
+ final Output out = new Output(buffers);
+ for (int offset = 0; offset < dataSize;) {
+ final int randomLength = random.nextInt(4 * max);
+ final int length = Math.min(randomLength, dataSize - offset);
+ LOG.info("{}: offset = {}, length = {}", name, offset, length);
+ final ByteBuffer b = ByteBuffer.wrap(data, offset, length);
+ final DataStreamReply writeReply = out.writeAsync(b).get();
+ assertReply(writeReply, length, null);
+ offset += length;
+ }
+
+ // close
+ final DataStreamReply closeReply = executePutBlockClose(
+ PUT_BLOCK_PROTO, max, out).get();
+ assertReply(closeReply, 0, PUT_BLOCK_PROTO);
+
+ // check output
+ final ByteBuf outBuf = out.getOutBuf();
+ LOG.info("outBuf = {}", outBuf);
+ Assert.assertEquals(dataSize, outBuf.readableBytes());
+ for (int i = 0; i < dataSize; i++) {
+ Assert.assertEquals(data[i], outBuf.readByte());
+ }
+ outBuf.release();
+ }
+
+ static void assertReply(DataStreamReply reply, int byteWritten,
+ ContainerCommandRequestProto proto) {
+ Assert.assertTrue(reply.isSuccess());
+ Assert.assertEquals(byteWritten, reply.getBytesWritten());
+ Assert.assertEquals(proto, ((Reply)reply).getPutBlockRequest());
+ }
+
+ static class Output implements DataStreamOutput {
+ private final Buffers buffers;
+ private final ByteBuf outBuf = Unpooled.buffer();
+ private final WriteMethod writeMethod = src -> {
+ final int remaining = src.remaining();
+ outBuf.writeBytes(src);
+ return remaining;
+ };
+
+ Output(Buffers buffers) {
+ this.buffers = buffers;
+ }
+
+ ByteBuf getOutBuf() {
+ return outBuf;
+ }
+
+ @Override
+ public CompletableFuture<DataStreamReply> writeAsync(
+ ByteBuffer src, WriteOption... writeOptions) {
+ final int written;
+ try {
+ written = writeBuffers(
+ ReferenceCountedObject.wrap(src, () -> { }, () -> { }),
+ buffers, writeMethod);
+ } catch (IOException e) {
+ return completeExceptionally(e);
+ }
+ if (WriteOption.containsOption(writeOptions, StandardWriteOption.CLOSE)) {
+ return closeAsync();
+ }
+ return CompletableFuture.completedFuture(
+ new Reply(true, written));
+ }
+
+ @Override
+ public CompletableFuture<DataStreamReply> closeAsync() {
+ final ContainerCommandRequestProto putBlockRequest;
+ try {
+ putBlockRequest = closeBuffers(buffers, writeMethod);
+ } catch (IOException e) {
+ return completeExceptionally(e);
+ }
+ return CompletableFuture.completedFuture(
+ new Reply(true, 0, putBlockRequest));
+ }
+
+ @Override
+ public CompletableFuture<DataStreamReply> writeAsync(
+ FilePositionCount filePositionCount, WriteOption... writeOptions) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public CompletableFuture<RaftClientReply> getRaftClientReplyFuture() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WritableByteChannel getWritableByteChannel() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ static class Reply implements DataStreamReply {
+ private final boolean success;
+ private final long bytesWritten;
+ private final ContainerCommandRequestProto putBlockRequest;
+
+ Reply(boolean success, long bytesWritten) {
+ this(success, bytesWritten, null);
+ }
+
+ Reply(boolean success, long bytesWritten,
+ ContainerCommandRequestProto putBlockRequest) {
+ this.success = success;
+ this.bytesWritten = bytesWritten;
+ this.putBlockRequest = putBlockRequest;
+ }
+
+ ContainerCommandRequestProto getPutBlockRequest() {
+ return putBlockRequest;
+ }
+
+ @Override
+ public boolean isSuccess() {
+ return success;
+ }
+
+ @Override
+ public long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ @Override
+ public Collection<CommitInfoProto> getCommitInfos() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ClientId getClientId() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataStreamPacketHeaderProto.Type getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getStreamId() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getStreamOffset() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getDataLength() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ static CompletableFuture<DataStreamReply> completeExceptionally(Throwable t) {
+ final CompletableFuture<DataStreamReply> f = new CompletableFuture<>();
+ f.completeExceptionally(t);
+ return f;
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 65f7348740..c8a0115a80 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -120,7 +120,7 @@ public class TestBlockDataStreamOutput {
objectStore.getVolume(volumeName).createBucket(bucketName);
}
- private String getKeyName() {
+ static String getKeyName() {
return UUID.randomUUID().toString();
}
@@ -158,13 +158,11 @@ public class TestBlockDataStreamOutput {
testWriteWithFailure(blockSize + 50);
}
- private void testWrite(int dataLength) throws Exception {
+ static void testWrite(int dataLength) throws Exception {
String keyName = getKeyName();
OzoneDataStreamOutput key = createKey(
keyName, ReplicationType.RATIS, dataLength);
- byte[] data =
- ContainerTestHelper.getFixedLengthString(keyString, dataLength)
- .getBytes(UTF_8);
+ final byte[] data = ContainerTestHelper.generateData(dataLength, false);
key.write(ByteBuffer.wrap(data));
// now close the stream, It will update the key length.
key.close();
@@ -221,14 +219,14 @@ public class TestBlockDataStreamOutput {
}
- private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
+ static OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
long size) throws Exception {
return TestHelper.createStreamKey(
keyName, type, size, objectStore, volumeName, bucketName);
}
- private void validateData(String keyName, byte[] data) throws Exception {
- TestHelper
- .validateData(keyName, data, objectStore, volumeName, bucketName);
+ static void validateData(String keyName, byte[] data) throws Exception {
+ TestHelper.validateData(
+ keyName, data, objectStore, volumeName, bucketName);
}
diff --git a/pom.xml b/pom.xml
index 54b89bb2a9..72b33a0237 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<declared.ozone.version>${ozone.version}</declared.ozone.version>
<!-- Apache Ratis version -->
- <ratis.version>2.3.0-da5d868-SNAPSHOT</ratis.version>
+ <ratis.version>2.3.0-6742a4e-SNAPSHOT</ratis.version>
<!-- Apache Ratis thirdparty version -->
<ratis.thirdparty.version>0.7.0</ratis.thirdparty.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 30/35: HDDS-6388. [Ozone-Streaming] Streaming write support both pipeline model and star model (#3145)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0d9b3e0e2720bbf9a76fa8bfd15c440b78a97dfd
Author: micah zhao <mi...@tencent.com>
AuthorDate: Wed Mar 2 23:26:48 2022 +0800
HDDS-6388. [Ozone-Streaming] Streaming write support both pipeline model and star model (#3145)
---
.../org/apache/hadoop/hdds/scm/OzoneClientConfig.java | 15 +++++++++++++++
.../hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 13 ++++++++++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index ffc4a3ef5a..fd3ab4751f 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -89,6 +89,13 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private long streamWindowSize = 64 * 1024 * 1024;
+ @Config(key = "datastream.pipeline.mode",
+ defaultValue = "true",
+ description = "Streaming write support both pipeline mode(datanode1->" +
+ "datanode2->datanode3) and star mode(datanode1->datanode2, " +
+ "datanode1->datanode3). By default we use pipeline mode.",
+ tags = ConfigTag.CLIENT)
+ private boolean datastreamPipelineMode = true;
@Config(key = "stream.buffer.increment",
defaultValue = "0B",
@@ -355,4 +362,12 @@ public class OzoneClientConfig {
public int getEcReconstructStripeReadPoolLimit() {
return ecReconstructStripeReadPoolLimit;
}
+
+ public boolean isDatastreamPipelineMode() {
+ return datastreamPipelineMode;
+ }
+
+ public void setDatastreamPipelineMode(boolean datastreamPipelineMode) {
+ this.datastreamPipelineMode = datastreamPipelineMode;
+ }
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 8b3e32cf41..3df5eb0e12 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -137,6 +137,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
private XceiverClientMetrics metrics;
// buffers for which putBlock is yet to be executed
private List<StreamBuffer> buffersForPutBlock;
+ private boolean isDatastreamPipelineMode;
/**
* Creates a new BlockDataStreamOutput.
*
@@ -154,6 +155,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
) throws IOException {
this.xceiverClientFactory = xceiverClientManager;
this.config = config;
+ this.isDatastreamPipelineMode = config.isDatastreamPipelineMode();
this.blockID = new AtomicReference<>(blockID);
KeyValue keyValue =
KeyValue.newBuilder().setKey("TYPE").setValue("KEY").build();
@@ -203,9 +205,14 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
ContainerCommandRequestMessage message =
ContainerCommandRequestMessage.toMessage(builder.build(), null);
- return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
- .stream(message.getContent().asReadOnlyByteBuffer(),
- getRoutingTable(pipeline));
+ if (isDatastreamPipelineMode) {
+ return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
+ .stream(message.getContent().asReadOnlyByteBuffer(),
+ getRoutingTable(pipeline));
+ } else {
+ return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
+ .stream(message.getContent().asReadOnlyByteBuffer());
+ }
}
public RoutingTable getRoutingTable(Pipeline pipeline) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 25/35: HDDS-6298. Add XceiverServerRatis stream config (#3070)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4fa81b239cd4d3989681ad50e000d3486acbeb9a
Author: hao guo <gu...@360.cn>
AuthorDate: Fri Feb 11 11:25:51 2022 +0800
HDDS-6298. Add XceiverServerRatis stream config (#3070)
---
.../src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index 604aaf4b93..d044026322 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -325,6 +325,10 @@ public final class RatisHelper {
return key.startsWith(NettyConfigKeys.DataStream.PREFIX);
}
+ private static boolean isStreamClientConfig(String key) {
+ return key.startsWith(RaftClientConfigKeys.DataStream.PREFIX);
+ }
+
/**
* Set all server properties matching with prefix
* {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in
@@ -339,7 +343,8 @@ public final class RatisHelper {
getDatanodeRatisPrefixProps(ozoneConf);
ratisServerConf.forEach((key, val) -> {
// Exclude ratis client configuration.
- if (!isClientConfig(key)) {
+ if (isNettyStreamConfig(key) || isStreamClientConfig(key) ||
+ !isClientConfig(key)) {
raftProperties.set(key, val);
}
});
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 27/35: HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE (#3060)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c13128155bedc282dd631eac833aefc7fe895a86
Author: hao guo <gu...@360.cn>
AuthorDate: Tue Feb 15 19:22:13 2022 +0800
HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE (#3060)
---
.../java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 9ac43300f7..8dd9e6b50e 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -318,7 +318,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// the bufferFull condition in async write path.
long streamWindow = config.getStreamWindowSize() / config
.getDataStreamMinPacketSize();
- if (!bufferList.isEmpty() && bufferList.size() % boundary == 0) {
+ if (!bufferList.isEmpty() && bufferList.size() % boundary == 0 &&
+ buffersForPutBlock != null && !buffersForPutBlock.isEmpty()) {
updateFlushLength();
executePutBlock(false, false);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 04/35: HDDS-5480. [Ozone-Streaming] Client and server should support stream setup. (#2452)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 14856e88082a000981da9142f3610e4673d2c95c
Author: micah zhao <mi...@tencent.com>
AuthorDate: Wed Jul 28 20:22:53 2021 +0800
HDDS-5480. [Ozone-Streaming] Client and server should support stream setup. (#2452)
---
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +++--
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 15 ++++++--
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 18 +++++++++
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../helpers/ContainerCommandRequestPBHelper.java | 1 +
.../common/src/main/resources/ozone-default.xml | 20 ++++++++++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 3 +-
.../transport/server/ratis/XceiverServerRatis.java | 43 +++++++++++++++++++++-
.../ozone/container/keyvalue/KeyValueHandler.java | 33 +++++++++++++++++
.../keyvalue/impl/ChunkManagerDispatcher.java | 6 +++
.../keyvalue/impl/FilePerBlockStrategy.java | 8 ++++
.../keyvalue/interfaces/ChunkManager.java | 5 +++
.../container/common/TestDatanodeStateMachine.java | 6 ++-
.../hdds/conf/DatanodeRatisServerConfig.java | 35 ++++++++++++++++++
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../ozone/container/common/TestEndPoint.java | 4 ++
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 1 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 3 ++
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +
.../server/TestSecureContainerServer.java | 2 +
23 files changed, 213 insertions(+), 15 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index 25826f3e23..78a0eeb7c5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -273,8 +273,10 @@ public class DatanodeDetails extends NodeImpl implements
return port;
}
}
- // if no separate admin/server port, return single Ratis one for compat
- if (name == Name.RATIS_ADMIN || name == Name.RATIS_SERVER) {
+ // if no separate admin/server/datastream port, return single Ratis one for
+ // compat
+ if (name == Name.RATIS_ADMIN || name == Name.RATIS_SERVER ||
+ name == Name.RATIS_DATASTREAM) {
return getPort(Name.RATIS);
}
return null;
@@ -784,7 +786,8 @@ public class DatanodeDetails extends NodeImpl implements
* Ports that are supported in DataNode.
*/
public enum Name {
- STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER;
+ STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER,
+ RATIS_DATASTREAM;
public static final Set<Name> ALL_PORTS = ImmutableSet.copyOf(
Name.values());
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index 8a6193d329..e93eb951ee 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -44,6 +44,7 @@ import org.apache.ratis.client.RaftClient;
import org.apache.ratis.client.RaftClientConfigKeys;
import org.apache.ratis.conf.Parameters;
import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.datastream.SupportedDataStreamType;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
@@ -119,7 +120,9 @@ public final class RatisHelper {
.setId(toRaftPeerId(dn))
.setAddress(toRaftPeerAddress(dn, Port.Name.RATIS_SERVER))
.setAdminAddress(toRaftPeerAddress(dn, Port.Name.RATIS_ADMIN))
- .setClientAddress(toRaftPeerAddress(dn, Port.Name.RATIS));
+ .setClientAddress(toRaftPeerAddress(dn, Port.Name.RATIS))
+ .setDataStreamAddress(
+ toRaftPeerAddress(dn, Port.Name.RATIS_DATASTREAM));
}
private static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
@@ -173,6 +176,7 @@ public final class RatisHelper {
ConfigurationSource ozoneConfiguration) throws IOException {
return newRaftClient(rpcType,
toRaftPeerId(pipeline.getLeaderNode()),
+ toRaftPeer(pipeline.getFirstNode()),
newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
pipeline.getNodes()), retryPolicy, tlsConfig, ozoneConfiguration);
}
@@ -192,7 +196,7 @@ public final class RatisHelper {
public static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
ConfigurationSource configuration) {
- return newRaftClient(rpcType, leader.getId(),
+ return newRaftClient(rpcType, leader.getId(), leader,
newRaftGroup(Collections.singletonList(leader)), retryPolicy,
tlsConfig, configuration);
}
@@ -200,14 +204,14 @@ public final class RatisHelper {
public static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy,
ConfigurationSource ozoneConfiguration) {
- return newRaftClient(rpcType, leader.getId(),
+ return newRaftClient(rpcType, leader.getId(), leader,
newRaftGroup(Collections.singletonList(leader)), retryPolicy, null,
ozoneConfiguration);
}
@SuppressWarnings("checkstyle:ParameterNumber")
private static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,
- RaftGroup group, RetryPolicy retryPolicy,
+ RaftPeer primary, RaftGroup group, RetryPolicy retryPolicy,
GrpcTlsConfig tlsConfig, ConfigurationSource ozoneConfiguration) {
if (LOG.isTraceEnabled()) {
LOG.trace("newRaftClient: {}, leader={}, group={}",
@@ -221,6 +225,7 @@ public final class RatisHelper {
return RaftClient.newBuilder()
.setRaftGroup(group)
.setLeaderId(leader)
+ .setPrimaryDataStreamServer(primary)
.setProperties(properties)
.setParameters(setClientTlsConf(rpcType, tlsConfig))
.setRetryPolicy(retryPolicy)
@@ -278,6 +283,8 @@ public final class RatisHelper {
public static RaftProperties setRpcType(RaftProperties properties,
RpcType rpcType) {
RaftConfigKeys.Rpc.setType(properties, rpcType);
+ RaftConfigKeys.DataStream.setType(properties,
+ SupportedDataStreamType.NETTY);
return properties;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index ded7bb07d6..ee0556dd43 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -57,6 +57,12 @@ public final class OzoneConfigKeys {
public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
false;
+ public static final String DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT =
+ "dfs.container.ratis.datastream.random.port";
+ public static final boolean
+ DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT_DEFAULT =
+ false;
+
public static final String DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY =
"dfs.container.chunk.write.sync";
public static final boolean DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT = false;
@@ -79,6 +85,18 @@ public final class OzoneConfigKeys {
"dfs.container.ratis.server.port";
public static final int DFS_CONTAINER_RATIS_SERVER_PORT_DEFAULT = 9856;
+ /**
+ * Ratis Port where containers listen to datastream requests.
+ */
+ public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLE
+ = "dfs.container.ratis.datastream.enable";
+ public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT
+ = true;
+ public static final String DFS_CONTAINER_RATIS_DATASTREAM_PORT
+ = "dfs.container.ratis.datastream.port";
+ public static final int DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT
+ = 9855;
+
/**
* When set to true, allocate a random free port for ozone container, so that
* a mini cluster is able to launch multiple containers on a node.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
index 1c87f2bdeb..73aff9ac83 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -38,7 +38,8 @@ public enum DNAction implements AuditAction {
PUT_SMALL_FILE,
GET_SMALL_FILE,
CLOSE_CONTAINER,
- GET_COMMITTED_BLOCK_LENGTH;
+ GET_COMMITTED_BLOCK_LENGTH,
+ STREAM_INIT;
@Override
public String getAction() {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
index a13f164eec..4d7f0f37c4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerCommandRequestPBHelper.java
@@ -187,6 +187,7 @@ public final class ContainerCommandRequestPBHelper {
case GetSmallFile : return DNAction.GET_SMALL_FILE;
case CloseContainer : return DNAction.CLOSE_CONTAINER;
case GetCommittedBlockLength : return DNAction.GET_COMMITTED_BLOCK_LENGTH;
+ case StreamInit : return DNAction.STREAM_INIT;
default :
LOG.debug("Invalid command type - {}", cmdType);
return null;
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4db1fe4479..e0732fc4c6 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,6 +53,26 @@
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>The ipc port number of container.</description>
</property>
+ <property>
+ <name>dfs.container.ratis.datastream.enable</name>
+ <value>true</value>
+ <tag>OZONE, CONTAINER, RATIS, DATASTREAM</tag>
+ <description>If enable datastream ipc of container.</description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.datastream.port</name>
+ <value>9855</value>
+ <tag>OZONE, CONTAINER, RATIS, DATASTREAM</tag>
+ <description>The datastream port number of container.</description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.datastream.random.port</name>
+ <value>false</value>
+ <tag>OZONE, CONTAINER, RATIS, DATASTREAM</tag>
+ <description>Allocates a random free port for ozone container datastream.
+ This is used only while running unit tests.
+ </description>
+ </property>
<property>
<name>dfs.container.ipc.random.port</name>
<value>false</value>
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 8cf584d75f..3728a0b1f5 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@ -46,5 +46,6 @@ public enum ConfigTag {
DELETION,
HA,
BALANCER,
- UPGRADE
+ UPGRADE,
+ DATASTREAM
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 83ba104439..2e0f1de496 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -199,7 +199,8 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
boolean isWriteStage =
(cmdType == Type.WriteChunk && dispatcherContext != null
&& dispatcherContext.getStage()
- == DispatcherContext.WriteChunkStage.WRITE_DATA);
+ == DispatcherContext.WriteChunkStage.WRITE_DATA)
+ || (cmdType == Type.StreamInit);
boolean isWriteCommitStage =
(cmdType == Type.WriteChunk && dispatcherContext != null
&& dispatcherContext.getStage()
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index c8d715cc60..d69b64cce1 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -98,6 +98,7 @@ import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
+import org.apache.ratis.server.DataStreamServerRpc;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.RaftServerRpc;
@@ -129,6 +130,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private int serverPort;
private int adminPort;
private int clientPort;
+ private int dataStreamPort;
private final RaftServer server;
private final List<ThreadPoolExecutor> chunkExecutors;
private final ContainerDispatcher dispatcher;
@@ -148,6 +150,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
// Timeout used while calling submitRequest directly.
private long requestTimeout;
private boolean shouldDeleteRatisLogDirectory;
+ private boolean streamEnable;
private XceiverServerRatis(DatanodeDetails dd,
ContainerDispatcher dispatcher, ContainerController containerController,
@@ -157,6 +160,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
Objects.requireNonNull(dd, "id == null");
datanodeDetails = dd;
assignPorts();
+ this.streamEnable = conf.getBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT);
RaftProperties serverProperties = newRaftProperties();
this.context = context;
this.dispatcher = dispatcher;
@@ -213,6 +219,32 @@ public final class XceiverServerRatis implements XceiverServerSpi {
chunkExecutors, this, conf);
}
+ private void setUpRatisStream(RaftProperties properties) {
+ // set the datastream config
+ if (conf.getBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT_DEFAULT)) {
+ dataStreamPort = 0;
+ } else {
+ dataStreamPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT);
+ }
+ NettyConfigKeys.DataStream.setPort(properties, dataStreamPort);
+ int dataStreamAsyncRequestThreadPoolSize =
+ conf.getObject(DatanodeRatisServerConfig.class)
+ .getStreamRequestThreads();
+ RaftServerConfigKeys.DataStream.setAsyncRequestThreadPoolSize(properties,
+ dataStreamAsyncRequestThreadPoolSize);
+ int dataStreamWriteRequestThreadPoolSize =
+ conf.getObject(DatanodeRatisServerConfig.class)
+ .getStreamWriteThreads();
+ RaftServerConfigKeys.DataStream.setAsyncWriteThreadPoolSize(properties,
+ dataStreamWriteRequestThreadPoolSize);
+ }
+
+ @SuppressWarnings("checkstyle:methodlength")
private RaftProperties newRaftProperties() {
final RaftProperties properties = new RaftProperties();
@@ -231,6 +263,10 @@ public final class XceiverServerRatis implements XceiverServerSpi {
// set the configs enable and set the stateMachineData sync timeout
RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true);
+ if (streamEnable) {
+ setUpRatisStream(properties);
+ }
+
timeUnit = OzoneConfigKeys.
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit();
duration = conf.getTimeDuration(
@@ -491,7 +527,12 @@ public final class XceiverServerRatis implements XceiverServerSpi {
Port.Name.RATIS_ADMIN);
serverPort = getRealPort(serverRpc.getInetSocketAddress(),
Port.Name.RATIS_SERVER);
-
+ if (streamEnable) {
+ DataStreamServerRpc dataStreamServerRpc =
+ server.getDataStreamServerRpc();
+ dataStreamPort = getRealPort(dataStreamServerRpc.getInetSocketAddress(),
+ Port.Name.RATIS_DATASTREAM);
+ }
isStarted = true;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index d0a865fde3..44e5f38a90 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -99,6 +99,7 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuil
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getReadChunkResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getReadContainerResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getSuccessResponse;
+import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getSuccessResponseBuilder;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
@@ -224,6 +225,8 @@ public class KeyValueHandler extends Handler {
return handler.handleDeleteChunk(request, kvContainer);
case WriteChunk:
return handler.handleWriteChunk(request, kvContainer, dispatcherContext);
+ case StreamInit:
+ return handler.handleStreamInit(request, kvContainer, dispatcherContext);
case ListChunk:
return handler.handleUnsupportedOp(request);
case CompactChunk:
@@ -250,6 +253,36 @@ public class KeyValueHandler extends Handler {
return this.blockManager;
}
+ ContainerCommandResponseProto handleStreamInit(
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
+ if (!request.hasWriteChunk()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Malformed Write Chunk request. trace ID: {}",
+ request.getTraceID());
+ }
+ return malformedRequest(request);
+ }
+
+ String path = null;
+ try {
+ checkContainerOpen(kvContainer);
+
+ WriteChunkRequestProto writeChunk = request.getWriteChunk();
+ BlockID blockID = BlockID.getFromProtobuf(writeChunk.getBlockID());
+
+ path = chunkManager
+ .streamInit(kvContainer, blockID);
+
+ } catch (StorageContainerException ex) {
+ return ContainerUtils.logAndReturnError(LOG, ex, request);
+ }
+
+ return getSuccessResponseBuilder(request)
+ .setMessage(path)
+ .build();
+ }
+
/**
* Handles Create Container Request. If successful, adds the container to
* ContainerSet and sends an ICR to the SCM.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
index 763647313b..3e2ab46470 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
@@ -73,6 +73,12 @@ public class ChunkManagerDispatcher implements ChunkManager {
.writeChunk(container, blockID, info, data, dispatcherContext);
}
+ public String streamInit(Container container, BlockID blockID)
+ throws StorageContainerException {
+ return selectHandler(container)
+ .streamInit(container, blockID);
+ }
+
@Override
public void finishWriteChunks(KeyValueContainer kvContainer,
BlockData blockData) throws IOException {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
index 51cd5708d3..9efc6bc351 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
@@ -89,6 +89,14 @@ public class FilePerBlockStrategy implements ChunkManager {
container.getContainerData().getLayoutVersion() == FILE_PER_BLOCK);
}
+ @Override
+ public String streamInit(Container container, BlockID blockID)
+ throws StorageContainerException {
+ checkLayoutVersion(container);
+ File chunkFile = getChunkFile(container, blockID, null);
+ return chunkFile.getAbsolutePath();
+ }
+
@Override
public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
ChunkBuffer data, DispatcherContext dispatcherContext)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 15ff9d6b9d..ba06eebd69 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -104,6 +104,11 @@ public interface ChunkManager {
// no-op
}
+ default String streamInit(Container container, BlockID blockID)
+ throws StorageContainerException {
+ return null;
+ }
+
static long getBufferCapacityForChunkRead(ChunkInfo chunkInfo,
long defaultReadBufferCapacity) {
long bufferCapacity = 0;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 1337f28ad9..bb1145bb2b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -81,6 +81,8 @@ public class TestDatanodeStateMachine {
TimeUnit.MILLISECONDS);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true);
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
serverAddresses = new ArrayList<>();
scmServers = new ArrayList<>();
mockServers = new ArrayList<>();
@@ -215,7 +217,6 @@ public class TestDatanodeStateMachine {
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
datanodeDetails.setPort(port);
ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
-
try (DatanodeStateMachine stateMachine =
new DatanodeStateMachine(datanodeDetails, conf, null, null,
null)) {
@@ -424,6 +425,8 @@ public class TestDatanodeStateMachine {
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
+ DatanodeDetails.Port streamPort = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS_DATASTREAM, 0);
return DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID())
.setHostName("localhost")
@@ -431,6 +434,7 @@ public class TestDatanodeStateMachine {
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort)
+ .addPort(streamPort)
.build();
}
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
index 25ed4776b7..205d92e955 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
@@ -23,6 +23,7 @@ import org.apache.ratis.server.RaftServerConfigKeys;
import java.time.Duration;
import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE;
+import static org.apache.hadoop.hdds.conf.ConfigTag.DATASTREAM;
import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE;
import static org.apache.hadoop.hdds.conf.ConfigTag.RATIS;
@@ -123,6 +124,40 @@ public class DatanodeRatisServerConfig {
this.leaderNumPendingRequests = leaderNumPendingRequests;
}
+ @Config(key = "datastream.request.threads",
+ defaultValue = "20",
+ type = ConfigType.INT,
+ tags = {OZONE, DATANODE, RATIS, DATASTREAM},
+ description = "Maximum number of threads in the thread pool for " +
+ "datastream request."
+ )
+ private int streamRequestThreads;
+
+ public int getStreamRequestThreads() {
+ return streamRequestThreads;
+ }
+
+ public void setStreamRequestThreads(int streamRequestThreads) {
+ this.streamRequestThreads = streamRequestThreads;
+ }
+
+ @Config(key = "datastream.write.threads",
+ defaultValue = "20",
+ type = ConfigType.INT,
+ tags = {OZONE, DATANODE, RATIS, DATASTREAM},
+ description = "Maximum number of threads in the thread pool for " +
+ "datastream write."
+ )
+ private int streamWriteThreads;
+
+ public int getStreamWriteThreads() {
+ return streamWriteThreads;
+ }
+
+ public void setStreamWriteThreads(int streamWriteThreads) {
+ this.streamWriteThreads = streamWriteThreads;
+ }
+
@Config(key = "delete.ratis.log.directory",
defaultValue = "true",
type = ConfigType.BOOLEAN,
diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
index 3f6e9996d2..62e8b9a55e 100644
--- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
@@ -100,6 +100,8 @@ enum Type {
GetSmallFile = 16;
CloseContainer = 17;
GetCommittedBlockLength = 18;
+
+ StreamInit = 19;
}
@@ -396,7 +398,7 @@ enum ChecksumType {
message WriteChunkRequestProto {
required DatanodeBlockID blockID = 1;
- required ChunkInfo chunkData = 2;
+ optional ChunkInfo chunkData = 2;
optional bytes data = 3;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index c0ff646693..a5fff4797f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -145,6 +145,8 @@ public class TestEndPoint {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, 1000)) {
DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
OzoneContainer ozoneContainer = new OzoneContainer(
datanodeDetails, conf, getContext(datanodeDetails), null);
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
@@ -169,6 +171,8 @@ public class TestEndPoint {
true);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
true);
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
conf.setFromObject(new ReplicationConfig().setPort(0));
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, 1000)) {
diff --git a/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode2.xml b/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode2.xml
index 3d3302030d..040b515b9f 100644
--- a/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode2.xml
+++ b/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode2.xml
@@ -18,7 +18,7 @@
<configuration default="false" name="Datanode2" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.HddsDatanodeService" />
<module name="ozone-datanode" />
- <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml --set ozone.metadata.dirs=/tmp/datanode2 --set hdds.datanode.dir=/tmp/datanode2/storage --set hdds.datanode.http-address=127.0.0.1:10021 --set dfs.container.ratis.ipc=10022 --set dfs.container.ipc=10023 --set dfs.container.ratis.server.port=10024 --set dfs.container.ratis.admin.port=10025 --set hdds.datanode.replication.port=10026" />
+ <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml --set ozone.metadata.dirs=/tmp/datanode2 --set hdds.datanode.dir=/tmp/datanode2/storage --set hdds.datanode.http-address=127.0.0.1:10021 --set dfs.container.ratis.ipc=10022 --set dfs.container.ipc=10023 --set dfs.container.ratis.server.port=10024 --set dfs.container.ratis.admin.port=10025 --set hdds.datanode.replication.port=10026 --set dfs.container.ratis.datastream.port=10027" />
<option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
<extension name="coverage">
<pattern>
diff --git a/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode3.xml b/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode3.xml
index 10b6682a0e..6a3116e0fd 100644
--- a/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode3.xml
+++ b/hadoop-ozone/dev-support/intellij/runConfigurations/Datanode3.xml
@@ -18,7 +18,7 @@
<configuration default="false" name="Datanode3" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.HddsDatanodeService" />
<module name="ozone-datanode" />
- <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml --set ozone.metadata.dirs=/tmp/datanode3 --set hdds.datanode.dir=/tmp/datanode3/storage --set hdds.datanode.http-address=127.0.0.1:10031 --set dfs.container.ratis.ipc=10032 --set dfs.container.ipc=10033 --set dfs.container.ratis.server.port=10034 --set dfs.container.ratis.admin.port=10035 --set hdds.datanode.replication.port=10036" />
+ <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml --set ozone.metadata.dirs=/tmp/datanode3 --set hdds.datanode.dir=/tmp/datanode3/storage --set hdds.datanode.http-address=127.0.0.1:10031 --set dfs.container.ratis.ipc=10032 --set dfs.container.ipc=10033 --set dfs.container.ratis.server.port=10034 --set dfs.container.ratis.admin.port=10035 --set hdds.datanode.replication.port=10036 --set dfs.container.ratis.datastream.port=10037" />
<option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
<extension name="coverage">
<pattern>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 556a7487c5..b1d9a74209 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -318,6 +318,7 @@ public interface MiniOzoneCluster {
protected Optional<String> omId = Optional.empty();
protected Boolean randomContainerPort = true;
+ protected Boolean randomContainerStreamPort = true;
protected Optional<String> datanodeReservedSpace = Optional.empty();
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 6920dd4df9..a549767481 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -88,6 +88,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_D
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT;
@@ -918,6 +919,8 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
randomContainerPort);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
randomContainerPort);
+ conf.setBoolean(DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT,
+ randomContainerStreamPort);
conf.setFromObject(new ReplicationConfig().setPort(0));
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 2f4df638e0..c06647bbe5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -215,6 +215,8 @@ public class TestMiniOzoneCluster {
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true);
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
true);
+ ozoneConf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
List<DatanodeStateMachine> stateMachines = new ArrayList<>();
try {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index cd7c995544..cb2db30c10 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -216,6 +216,8 @@ public class TestSecureContainerServer {
DatanodeDetails dn, OzoneConfiguration conf) throws IOException {
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
dn.getPort(DatanodeDetails.Port.Name.RATIS).getValue());
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
final String dir = TEST_DIR + dn.getUuid();
conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir);
final ContainerDispatcher dispatcher = createDispatcher(dn,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 19/35: HDDS-6039. Define a minimum packet size during streaming writes. (#2883)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit ac9815d47c89a5e148e46487b9b821ef3f9ed132
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Tue Dec 21 12:22:30 2021 +0530
HDDS-6039. Define a minimum packet size during streaming writes. (#2883)
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 16 ++++++
.../hdds/scm/storage/BlockDataStreamOutput.java | 57 ++++++++++++++++------
.../hadoop/hdds/scm/storage/StreamBuffer.java | 15 +++++-
.../hdds/scm/storage/StreamCommitWatcher.java | 2 +-
.../client/io/BlockDataStreamOutputEntryPool.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 6 +++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 6 +++
.../client/rpc/TestBlockDataStreamOutput.java | 32 ++++++++++--
.../rpc/TestContainerStateMachineStream.java | 1 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 1 +
10 files changed, 118 insertions(+), 20 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index 4fad9df250..5249e53644 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -81,6 +81,14 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private long dataStreamBufferFlushSize = 16 * 1024 * 1024;
+ @Config(key = "datastream.min.packet.size",
+ defaultValue = "1MB",
+ type = ConfigType.SIZE,
+ description = "The maximum size of the ByteBuffer "
+ + "(used via ratis streaming)",
+ tags = ConfigTag.CLIENT)
+ private int dataStreamMinPacketSize = 1024 * 1024;
+
@Config(key = "stream.buffer.increment",
defaultValue = "0B",
type = ConfigType.SIZE,
@@ -259,6 +267,14 @@ public class OzoneClientConfig {
this.streamBufferMaxSize = streamBufferMaxSize;
}
+ public int getDataStreamMinPacketSize() {
+ return dataStreamMinPacketSize;
+ }
+
+ public void setDataStreamMinPacketSize(int dataStreamMinPacketSize) {
+ this.dataStreamMinPacketSize = dataStreamMinPacketSize;
+ }
+
public int getMaxRetryCount() {
return maxRetryCount;
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 6f5a54354a..9fb1340527 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.XceiverClientRatis;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -125,7 +127,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
private List<CompletableFuture<DataStreamReply>> futures = new ArrayList<>();
private final long syncSize = 0; // TODO: disk sync is disabled for now
private long syncPosition = 0;
-
+ private StreamBuffer currentBuffer;
+ private XceiverClientMetrics metrics;
/**
* Creates a new BlockDataStreamOutput.
*
@@ -172,6 +175,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
ioException = new AtomicReference<>(null);
checksum = new Checksum(config.getChecksumType(),
config.getBytesPerChecksum());
+ metrics = XceiverClientManager.getXceiverClientMetrics();
}
private DataStreamOutput setupStream(Pipeline pipeline) throws IOException {
@@ -257,27 +261,47 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
if (len == 0) {
return;
}
- int curLen = len;
- // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
- int maxBufferLen = config.getDataStreamMaxBufferSize();
- while (curLen > 0) {
- int writeLen = Math.min(curLen, maxBufferLen);
+ while (len > 0) {
+ allocateNewBufferIfNeeded();
+ int writeLen = Math.min(len, currentBuffer.length());
final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
+ currentBuffer.put(buf);
+ writeChunkIfNeeded();
off += writeLen;
- bufferList.add(buf);
- writeChunkToContainer(buf.duplicate());
- curLen -= writeLen;
writtenDataLength += writeLen;
+ len -= writeLen;
doFlushIfNeeded();
}
}
+ private void writeChunkIfNeeded() throws IOException {
+ if (currentBuffer.length()==0) {
+ writeChunk(currentBuffer);
+ currentBuffer = null;
+ }
+ }
+
+ private void writeChunk(StreamBuffer sb) throws IOException {
+ bufferList.add(sb);
+ ByteBuffer dup = sb.duplicate();
+ dup.position(0);
+ dup.limit(sb.position());
+ writeChunkToContainer(dup);
+ }
+
+ private void allocateNewBufferIfNeeded() {
+ if (currentBuffer==null) {
+ currentBuffer =
+ StreamBuffer.allocate(config.getDataStreamMinPacketSize());
+ }
+ }
+
private void doFlushIfNeeded() throws IOException {
Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
.getDataStreamMaxBufferSize());
long boundary = config.getDataStreamBufferFlushSize() / config
.getDataStreamMaxBufferSize();
- if (bufferList.size() % boundary == 0) {
+ if (!bufferList.isEmpty() && bufferList.size() % boundary == 0) {
updateFlushLength();
executePutBlock(false, false);
}
@@ -308,11 +332,10 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
int count = 0;
while (len > 0) {
final StreamBuffer buf = bufferList.get(count);
- final long writeLen = Math.min(buf.length(), len);
+ final long writeLen = Math.min(buf.position(), len);
final ByteBuffer duplicated = buf.duplicate();
- if (writeLen != buf.length()) {
- duplicated.limit(Math.toIntExact(len));
- }
+ duplicated.position(0);
+ duplicated.limit(buf.position());
writeChunkToContainer(duplicated);
len -= writeLen;
count++;
@@ -449,6 +472,11 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// This can be a partially filled chunk. Since we are flushing the buffer
// here, we just limit this buffer to the current position. So that next
// write will happen in new buffer
+
+ if (currentBuffer!=null) {
+ writeChunk(currentBuffer);
+ currentBuffer = null;
+ }
updateFlushLength();
executePutBlock(close, false);
} else if (close) {
@@ -584,6 +612,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
.setLen(effectiveChunkSize)
.setChecksumData(checksumData.getProtoBufMessage())
.build();
+ metrics.incrPendingContainerOpsMetrics(ContainerProtos.Type.WriteChunk);
if (LOG.isDebugEnabled()) {
LOG.debug("Writing chunk {} length {} at offset {}",
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
index f36019e2ae..5118ea5ead 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
@@ -27,7 +27,7 @@ public class StreamBuffer {
private final ByteBuffer buffer;
public StreamBuffer(ByteBuffer buffer) {
- this.buffer = buffer.asReadOnlyBuffer();
+ this.buffer = buffer;
}
public StreamBuffer(ByteBuffer buffer, int offset, int length) {
@@ -43,4 +43,17 @@ public class StreamBuffer {
return buffer.limit() - buffer.position();
}
+ public int position() {
+ return buffer.position();
+ }
+
+
+ public void put(StreamBuffer sb){
+ buffer.put(sb.buffer);
+ }
+
+ public static StreamBuffer allocate(int size){
+ return new StreamBuffer(ByteBuffer.allocate(size));
+ }
+
}
\ No newline at end of file
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
index 3a59d07571..9ae604e951 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
@@ -178,7 +178,7 @@ public class StreamCommitWatcher {
Preconditions.checkState(commitIndexMap.containsKey(index));
final List<StreamBuffer> buffers = commitIndexMap.remove(index);
final long length =
- buffers.stream().mapToLong(StreamBuffer::length).sum();
+ buffers.stream().mapToLong(StreamBuffer::position).sum();
totalAckDataLength += length;
// clear the future object from the future Map
final CompletableFuture<ContainerCommandResponseProto> remove =
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index e49b0b79ad..24a046f623 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -309,7 +309,7 @@ public class BlockDataStreamOutputEntryPool {
long computeBufferData() {
long totalDataLen =0;
for (StreamBuffer b : bufferList){
- totalDataLen += b.length();
+ totalDataLen += b.position();
}
return totalDataLen;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 501f9ce0fb..5c0e74f80b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -326,6 +326,7 @@ public interface MiniOzoneCluster {
protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferMaxSize = Optional.empty();
+ protected OptionalInt dataStreamMinPacketSize = OptionalInt.empty();
protected Optional<Long> blockSize = Optional.empty();
protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
protected boolean includeRecon = false;
@@ -569,6 +570,11 @@ public interface MiniOzoneCluster {
return this;
}
+ public Builder setDataStreamMinPacketSize(int size) {
+ dataStreamMinPacketSize = OptionalInt.of(size);
+ return this;
+ }
+
/**
* Sets the block size for stream buffer.
*
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 8f75659ce9..ce619aac59 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -667,6 +667,9 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!dataStreamMaxBufferSize.isPresent()) {
dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
}
+ if (!dataStreamMinPacketSize.isPresent()) {
+ dataStreamMinPacketSize = OptionalInt.of(chunkSize.get()/4);
+ }
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
@@ -688,6 +691,9 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
clientConfig.setDataStreamMaxBufferSize((int) Math.round(
streamBufferSizeUnit.get()
.toBytes(dataStreamMaxBufferSize.getAsInt())));
+ clientConfig.setDataStreamMinPacketSize((int) Math.round(
+ streamBufferSizeUnit.get()
+ .toBytes(dataStreamMinPacketSize.getAsInt())));
conf.setFromObject(clientConfig);
conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 5eb38a00de..c9242df8b1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -107,6 +107,7 @@ public class TestBlockDataStreamOutput {
.setDataStreamBufferFlushize(maxFlushSize)
.setDataStreamBufferMaxSize(chunkSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
+ .setDataStreamMinPacketSize(2*chunkSize/5)
.build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
@@ -193,7 +194,7 @@ public class TestBlockDataStreamOutput {
@Test
public void testPutBlockAtBoundary() throws Exception {
- int dataLength = 500;
+ int dataLength = 200;
XceiverClientMetrics metrics =
XceiverClientManager.getXceiverClientMetrics();
long putBlockCount = metrics.getContainerOpCountMetrics(
@@ -211,8 +212,8 @@ public class TestBlockDataStreamOutput {
metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
key.close();
- // Since data length is 500 , first putBlock will be at 400(flush boundary)
- // and the other at 500
+ // Since data length is 200 , first putBlock will be at 160(flush boundary)
+ // and the other at 200
Assert.assertTrue(
metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
== putBlockCount + 2);
@@ -230,4 +231,29 @@ public class TestBlockDataStreamOutput {
.validateData(keyName, data, objectStore, volumeName, bucketName);
}
+
+ @Test
+ public void testMinPacketSize() throws Exception {
+ String keyName = getKeyName();
+ XceiverClientMetrics metrics =
+ XceiverClientManager.getXceiverClientMetrics();
+ OzoneDataStreamOutput key = createKey(keyName, ReplicationType.RATIS, 0);
+ long writeChunkCount =
+ metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
+ byte[] data =
+ ContainerTestHelper.getFixedLengthString(keyString, chunkSize / 5)
+ .getBytes(UTF_8);
+ key.write(ByteBuffer.wrap(data));
+ // minPacketSize= 40, so first write of 20 wont trigger a writeChunk
+ Assert.assertEquals(writeChunkCount,
+ metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ key.write(ByteBuffer.wrap(data));
+ Assert.assertEquals(writeChunkCount + 1,
+ metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ // now close the stream, It will update the key length.
+ key.close();
+ String dataString = new String(data, UTF_8);
+ validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
+ }
+
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
index 3b17450376..f4c756bccd 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
@@ -119,6 +119,7 @@ public class TestContainerStateMachineStream {
conf.setQuietMode(false);
cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).setHbInterval(200)
+ .setDataStreamMinPacketSize(1024)
.build();
cluster.waitForClusterToBeReady();
cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index fb50b81f18..574fed7a5b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -200,6 +200,7 @@ public abstract class TestOzoneRpcClientAbstract {
.setTotalPipelineNumLimit(10)
.setScmId(scmId)
.setClusterId(clusterId)
+ .setDataStreamMinPacketSize(1024)
.build();
cluster.waitForClusterToBeReady();
ozClient = OzoneClientFactory.getRpcClient(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 28/35: HDDS-6229. [Ozone-Streaming] Data Channel abstraction on datanode (#3023)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit b2810432208965ef08f2d5f7e68af21214f859e8
Author: hao guo <gu...@360.cn>
AuthorDate: Tue Feb 15 23:35:46 2022 +0800
HDDS-6229. [Ozone-Streaming] Data Channel abstraction on datanode (#3023)
---
.../keyvalue/impl/KeyValueStreamDataChannel.java | 56 ++--------------------
...DataChannel.java => StreamDataChannelBase.java} | 39 ++++++++-------
2 files changed, 27 insertions(+), 68 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
index c0570f5d4d..14ead4ea86 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
@@ -22,69 +22,21 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.ratis.statemachine.StateMachine;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
/**
* This class is used to get the DataChannel for streaming.
*/
-class KeyValueStreamDataChannel implements StateMachine.DataChannel {
- private final RandomAccessFile randomAccessFile;
- private final File file;
-
- private final ContainerData containerData;
- private final ContainerMetrics metrics;
-
+class KeyValueStreamDataChannel extends StreamDataChannelBase {
KeyValueStreamDataChannel(File file, ContainerData containerData,
ContainerMetrics metrics)
throws StorageContainerException {
- try {
- this.file = file;
- this.randomAccessFile = new RandomAccessFile(file, "rw");
- } catch (FileNotFoundException e) {
- throw new StorageContainerException("BlockFile not exists with " +
- "container Id " + containerData.getContainerID() +
- " file " + file.getAbsolutePath(),
- ContainerProtos.Result.IO_EXCEPTION);
- }
- this.containerData = containerData;
- this.metrics = metrics;
- }
-
- @Override
- public void force(boolean metadata) throws IOException {
- randomAccessFile.getChannel().force(metadata);
- }
-
- @Override
- public int write(ByteBuffer src) throws IOException {
- int writeBytes = randomAccessFile.getChannel().write(src);
- metrics
- .incContainerBytesStats(ContainerProtos.Type.StreamWrite, writeBytes);
- containerData.updateWriteStats(writeBytes, false);
- return writeBytes;
- }
-
- @Override
- public boolean isOpen() {
- return randomAccessFile.getChannel().isOpen();
- }
-
- @Override
- public void close() throws IOException {
- randomAccessFile.close();
+ super(file, containerData, metrics);
}
@Override
- public String toString() {
- return "KeyValueStreamDataChannel{" +
- "File=" + file.getAbsolutePath() +
- ", containerID=" + containerData.getContainerID() +
- '}';
+ ContainerProtos.Type getType() {
+ return ContainerProtos.Type.StreamWrite;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
similarity index 77%
copy from hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
copy to hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
index c0570f5d4d..b31e2ccbf4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
@@ -29,19 +29,21 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
/**
- * This class is used to get the DataChannel for streaming.
+ * For write state machine data.
*/
-class KeyValueStreamDataChannel implements StateMachine.DataChannel {
+abstract class StreamDataChannelBase implements StateMachine.DataChannel {
private final RandomAccessFile randomAccessFile;
+
private final File file;
private final ContainerData containerData;
private final ContainerMetrics metrics;
- KeyValueStreamDataChannel(File file, ContainerData containerData,
- ContainerMetrics metrics)
+ StreamDataChannelBase(File file, ContainerData containerData,
+ ContainerMetrics metrics)
throws StorageContainerException {
try {
this.file = file;
@@ -56,23 +58,20 @@ class KeyValueStreamDataChannel implements StateMachine.DataChannel {
this.metrics = metrics;
}
- @Override
- public void force(boolean metadata) throws IOException {
- randomAccessFile.getChannel().force(metadata);
+ abstract ContainerProtos.Type getType();
+
+ private FileChannel getChannel() {
+ return randomAccessFile.getChannel();
}
@Override
- public int write(ByteBuffer src) throws IOException {
- int writeBytes = randomAccessFile.getChannel().write(src);
- metrics
- .incContainerBytesStats(ContainerProtos.Type.StreamWrite, writeBytes);
- containerData.updateWriteStats(writeBytes, false);
- return writeBytes;
+ public final void force(boolean metadata) throws IOException {
+ getChannel().force(metadata);
}
@Override
- public boolean isOpen() {
- return randomAccessFile.getChannel().isOpen();
+ public final boolean isOpen() {
+ return getChannel().isOpen();
}
@Override
@@ -80,9 +79,17 @@ class KeyValueStreamDataChannel implements StateMachine.DataChannel {
randomAccessFile.close();
}
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ final int writeBytes = getChannel().write(src);
+ metrics.incContainerBytesStats(getType(), writeBytes);
+ containerData.updateWriteStats(writeBytes, false);
+ return writeBytes;
+ }
+
@Override
public String toString() {
- return "KeyValueStreamDataChannel{" +
+ return getClass().getSimpleName() + "{" +
"File=" + file.getAbsolutePath() +
", containerID=" + containerData.getContainerID() +
'}';
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 06/35: HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer to avoid buffer copying (#2557)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0790843af01b0e439c0f39c80f1c478cc68632c7
Author: micah zhao <mi...@tencent.com>
AuthorDate: Wed Aug 25 23:39:10 2021 +0800
HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer to avoid buffer copying (#2557)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 290 +++------------------
.../hdds/scm/storage/StreamCommitWatcher.java | 166 ++++++++++++
.../client/io/BlockDataStreamOutputEntry.java | 33 +--
.../client/io/BlockDataStreamOutputEntryPool.java | 20 --
.../ozone/client/io/KeyDataStreamOutput.java | 29 +--
5 files changed, 211 insertions(+), 327 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index f658df1af9..39ec2f9219 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -36,21 +36,18 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
-import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.ratis.client.api.DataStreamOutput;
import org.apache.ratis.io.StandardWriteOption;
import org.apache.ratis.protocol.DataStreamReply;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
@@ -93,7 +90,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
private int chunkIndex;
private final AtomicLong chunkOffset = new AtomicLong();
- private final BufferPool bufferPool;
// The IOException will be set by response handling thread in case there is an
// exception received in the response. If the exception is set, the next
// request will fail upfront.
@@ -106,28 +102,16 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
// effective data write attempted so far for the block
private long writtenDataLength;
- // List containing buffers for which the putBlock call will
- // update the length in the datanodes. This list will just maintain
- // references to the buffers in the BufferPool which will be cleared
- // when the watchForCommit acknowledges a putBlock logIndex has been
- // committed on all datanodes. This list will be a place holder for buffers
- // which got written between successive putBlock calls.
- private List<ChunkBuffer> bufferList;
-
// This object will maintain the commitIndexes and byteBufferList in order
// Also, corresponding to the logIndex, the corresponding list of buffers will
// be released from the buffer pool.
- private final CommitWatcher commitWatcher;
+ private final StreamCommitWatcher commitWatcher;
private final List<DatanodeDetails> failedServers;
private final Checksum checksum;
//number of buffers used before doing a flush/putBlock.
private int flushPeriod;
- //bytes remaining to write in the current buffer.
- private int currentBufferRemaining;
- //current buffer allocated to write
- private ChunkBuffer currentBuffer;
private final Token<? extends TokenIdentifier> token;
private final DataStreamOutput out;
private CompletableFuture<DataStreamReply> dataStreamCloseReply;
@@ -141,13 +125,11 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
* @param blockID block ID
* @param xceiverClientManager client manager that controls client
* @param pipeline pipeline where block will be written
- * @param bufferPool pool of buffers
*/
public BlockDataStreamOutput(
BlockID blockID,
XceiverClientFactory xceiverClientManager,
Pipeline pipeline,
- BufferPool bufferPool,
OzoneClientConfig config,
Token<? extends TokenIdentifier> token
) throws IOException {
@@ -163,11 +145,8 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
(XceiverClientRatis)xceiverClientManager.acquireClient(pipeline);
// Alternatively, stream setup can be delayed till the first chunk write.
this.out = setupStream();
- this.bufferPool = bufferPool;
this.token = token;
- //number of buffers used before doing a flush
- refreshCurrentBuffer(bufferPool);
flushPeriod = (int) (config.getStreamBufferFlushSize() / config
.getStreamBufferSize());
@@ -178,8 +157,7 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
// A single thread executor handle the responses of async requests
responseExecutor = Executors.newSingleThreadExecutor();
- commitWatcher = new CommitWatcher(bufferPool, xceiverClient);
- bufferList = null;
+ commitWatcher = new StreamCommitWatcher(xceiverClient);
totalDataFlushedLength = 0;
writtenDataLength = 0;
failedServers = new ArrayList<>(0);
@@ -209,20 +187,10 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
.stream(message.getContent().asReadOnlyByteBuffer());
}
- private void refreshCurrentBuffer(BufferPool pool) {
- currentBuffer = pool.getCurrentBuffer();
- currentBufferRemaining =
- currentBuffer != null ? currentBuffer.remaining() : 0;
- }
-
public BlockID getBlockID() {
return blockID.get();
}
- public long getTotalAckDataLength() {
- return commitWatcher.getTotalAckDataLength();
- }
-
public long getWrittenDataLength() {
return writtenDataLength;
}
@@ -236,82 +204,29 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
return xceiverClient;
}
- @VisibleForTesting
- public long getTotalDataFlushedLength() {
- return totalDataFlushedLength;
- }
-
- @VisibleForTesting
- public BufferPool getBufferPool() {
- return bufferPool;
- }
-
public IOException getIoException() {
return ioException.get();
}
- @VisibleForTesting
- public Map<Long, List<ChunkBuffer>> getCommitIndex2flushedDataMap() {
- return commitWatcher.getCommitIndex2flushedDataMap();
- }
-
@Override
- public void write(ByteBuf b) throws IOException {
+ public void write(ByteBuf buf) throws IOException {
checkOpen();
- if (b == null) {
+ if (buf == null) {
throw new NullPointerException();
}
- int off = b.readerIndex();
- int len = b.readableBytes();
-
- while (len > 0) {
- allocateNewBufferIfNeeded();
- final int writeLen = Math.min(currentBufferRemaining, len);
- // TODO: avoid buffer copy here
- currentBuffer.put(b.nioBuffer(off, writeLen));
- currentBufferRemaining -= writeLen;
- writeChunkIfNeeded();
- off += writeLen;
- len -= writeLen;
- writtenDataLength += writeLen;
- doFlushOrWatchIfNeeded();
- }
- }
-
- private void writeChunkIfNeeded() throws IOException {
- if (currentBufferRemaining == 0) {
- writeChunk(currentBuffer);
- }
- }
-
- private void doFlushOrWatchIfNeeded() throws IOException {
- if (currentBufferRemaining == 0) {
- if (bufferPool.getNumberOfUsedBuffers() % flushPeriod == 0) {
- updateFlushLength();
- executePutBlock(false, false);
- }
- // Data in the bufferPool can not exceed streamBufferMaxSize
- if (bufferPool.getNumberOfUsedBuffers() == bufferPool.getCapacity()) {
- handleFullBuffer();
- }
+ final int len = buf.readableBytes();
+ if (len == 0) {
+ return;
}
- }
+ writeChunkToContainer(buf);
- private void allocateNewBufferIfNeeded() {
- if (currentBufferRemaining == 0) {
- currentBuffer = bufferPool.allocateBuffer(config.getBufferIncrement());
- currentBufferRemaining = currentBuffer.remaining();
- }
+ writtenDataLength += len;
}
private void updateFlushLength() {
totalDataFlushedLength = writtenDataLength;
}
- private boolean isBufferPoolFull() {
- return bufferPool.computeBufferData() == config.getStreamBufferMaxSize();
- }
-
/**
* Will be called on the retryPath in case closedContainerException/
* TimeoutException.
@@ -319,70 +234,9 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
* @throws IOException if error occurred
*/
- // In this case, the data is already cached in the currentBuffer.
+ // TODO: We need add new retry policy without depend on bufferPool.
public void writeOnRetry(long len) throws IOException {
- if (len == 0) {
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrying write length {} for blockID {}", len, blockID);
- }
- Preconditions.checkArgument(len <= config.getStreamBufferMaxSize());
- int count = 0;
- while (len > 0) {
- ChunkBuffer buffer = bufferPool.getBuffer(count);
- long writeLen = Math.min(buffer.position(), len);
- if (!buffer.hasRemaining()) {
- writeChunk(buffer);
- }
- len -= writeLen;
- count++;
- writtenDataLength += writeLen;
- // we should not call isBufferFull/shouldFlush here.
- // The buffer might already be full as whole data is already cached in
- // the buffer. We should just validate
- // if we wrote data of size streamBufferMaxSize/streamBufferFlushSize to
- // call for handling full buffer/flush buffer condition.
- if (writtenDataLength % config.getStreamBufferFlushSize() == 0) {
- // reset the position to zero as now we will be reading the
- // next buffer in the list
- updateFlushLength();
- executePutBlock(false, false);
- }
- if (writtenDataLength == config.getStreamBufferMaxSize()) {
- handleFullBuffer();
- }
- }
- }
- /**
- * This is a blocking call. It will wait for the flush till the commit index
- * at the head of the commitIndex2flushedDataMap gets replicated to all or
- * majority.
- * @throws IOException
- */
- private void handleFullBuffer() throws IOException {
- try {
- checkOpen();
- if (!commitWatcher.getFutureMap().isEmpty()) {
- waitOnFlushFutures();
- }
- } catch (ExecutionException e) {
- handleExecutionException(e);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- handleInterruptedException(ex, true);
- }
- watchForCommit(true);
- }
-
-
- // It may happen that once the exception is encountered , we still might
- // have successfully flushed up to a certain index. Make sure the buffers
- // only contain data which have not been sufficiently replicated
- private void adjustBuffersOnException() {
- commitWatcher.releaseBuffersOnException();
- refreshCurrentBuffer(bufferPool);
}
/**
@@ -397,7 +251,8 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
checkOpen();
try {
XceiverClientReply reply = bufferFull ?
- commitWatcher.watchOnFirstIndex() : commitWatcher.watchOnLastIndex();
+ commitWatcher.streamWatchOnFirstIndex() :
+ commitWatcher.streamWatchOnLastIndex();
if (reply != null) {
List<DatanodeDetails> dnList = reply.getDatanodes();
if (!dnList.isEmpty()) {
@@ -412,7 +267,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
setIoException(ioe);
throw getIoException();
}
- refreshCurrentBuffer(bufferPool);
}
@@ -426,22 +280,7 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
boolean force) throws IOException {
checkOpen();
long flushPos = totalDataFlushedLength;
- final List<ChunkBuffer> byteBufferList;
- if (!force) {
- Preconditions.checkNotNull(bufferList);
- byteBufferList = bufferList;
- bufferList = null;
- Preconditions.checkNotNull(byteBufferList);
- } else {
- byteBufferList = null;
- }
-
- try {
- CompletableFuture.allOf(futures.toArray(EMPTY_FUTURE_ARRAY)).get();
- } catch (Exception e) {
- LOG.warn("Failed to write all chunks through stream: " + e);
- throw new IOException(e);
- }
+ flush();
if (close) {
dataStreamCloseReply = out.closeAsync();
}
@@ -471,15 +310,12 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding index " + asyncReply.getLogIndex() + " commitMap size "
- + commitWatcher.getCommitInfoMapSize() + " flushLength "
- + flushPos + " numBuffers " + byteBufferList.size()
- + " blockID " + blockID + " bufferPool size" + bufferPool
- .getSize() + " currentBufferIndex " + bufferPool
- .getCurrentBufferIndex());
+ + commitWatcher.getCommitInfoSetSize() + " flushLength "
+ + flushPos + " blockID " + blockID);
}
// for standalone protocol, logIndex will always be 0.
- commitWatcher
- .updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
+ commitWatcher.updateCommitInfoSet(
+ asyncReply.getLogIndex());
}
return e;
}, responseExecutor).exceptionally(e -> {
@@ -503,36 +339,12 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
@Override
public void flush() throws IOException {
- if (xceiverClientFactory != null && xceiverClient != null
- && bufferPool != null && bufferPool.getSize() > 0
- && (!config.isStreamBufferFlushDelay() ||
- writtenDataLength - totalDataFlushedLength
- >= config.getStreamBufferSize())) {
- try {
- handleFlush(false);
- } catch (ExecutionException e) {
- // just set the exception here as well in order to maintain sanctity of
- // ioException field
- handleExecutionException(e);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- handleInterruptedException(ex, true);
- }
- }
- }
-
- private void writeChunk(ChunkBuffer buffer)
- throws IOException {
- // This data in the buffer will be pushed to datanode and a reference will
- // be added to the bufferList. Once putBlock gets executed, this list will
- // be marked null. Hence, during first writeChunk call after every putBlock
- // call or during the first call to writeChunk here, the list will be null.
-
- if (bufferList == null) {
- bufferList = new ArrayList<>();
+ try {
+ CompletableFuture.allOf(futures.toArray(EMPTY_FUTURE_ARRAY)).get();
+ } catch (Exception e) {
+ LOG.warn("Failed to write all chunks through stream: " + e);
+ throw new IOException(e);
}
- bufferList.add(buffer);
- writeChunkToContainer(buffer.duplicate(0, buffer.position()));
}
/**
@@ -543,11 +355,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
checkOpen();
// flush the last chunk data residing on the currentBuffer
if (totalDataFlushedLength < writtenDataLength) {
- refreshCurrentBuffer(bufferPool);
- Preconditions.checkArgument(currentBuffer.position() > 0);
- if (currentBuffer.hasRemaining()) {
- writeChunk(currentBuffer);
- }
// This can be a partially filled chunk. Since we are flushing the buffer
// here, we just limit this buffer to the current position. So that next
// write will happen in new buffer
@@ -570,8 +377,7 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
@Override
public void close() throws IOException {
- if (xceiverClientFactory != null && xceiverClient != null
- && bufferPool != null && bufferPool.getSize() > 0) {
+ if (xceiverClientFactory != null && xceiverClient != null) {
try {
handleFlush(true);
dataStreamCloseReply.get();
@@ -583,10 +389,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
} finally {
cleanup(false);
}
- // TODO: Turn the below buffer empty check on when Standalone pipeline
- // is removed in the write path in tests
- // Preconditions.checkArgument(buffer.position() == 0);
- // bufferPool.checkBufferPoolEmpty();
}
}
@@ -638,10 +440,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
xceiverClientFactory = null;
xceiverClient = null;
commitWatcher.cleanup();
- if (bufferList != null) {
- bufferList.clear();
- }
- bufferList = null;
responseExecutor.shutdown();
}
@@ -655,7 +453,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
if (isClosed()) {
throw new IOException("BlockDataStreamOutput has been closed.");
} else if (getIoException() != null) {
- adjustBuffersOnException();
throw getIoException();
}
}
@@ -683,12 +480,11 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
* @throws OzoneChecksumException if there is an error while computing
* checksum
*/
- private void writeChunkToContainer(ChunkBuffer chunk) throws IOException {
- int effectiveChunkSize = chunk.remaining();
+ private void writeChunkToContainer(ByteBuf buf)
+ throws IOException {
+ ChecksumData checksumData = checksum.computeChecksum(buf.nioBuffer());
+ int effectiveChunkSize = buf.readableBytes();
final long offset = chunkOffset.getAndAdd(effectiveChunkSize);
- final ByteString data = chunk.toByteString(
- bufferPool.byteStringConversion());
- ChecksumData checksumData = checksum.computeChecksum(chunk);
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(blockID.get().getLocalID() + "_chunk_" + ++chunkIndex)
.setOffset(offset)
@@ -703,21 +499,22 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
CompletableFuture<DataStreamReply> future =
(needSync(offset + effectiveChunkSize) ?
- out.writeAsync(data.asReadOnlyByteBuffer(), StandardWriteOption.SYNC) :
- out.writeAsync(data.asReadOnlyByteBuffer()))
- .whenCompleteAsync((r, e) -> {
- if (e != null || !r.isSuccess()) {
- if (e == null) {
- e = new IOException("result is not success");
- }
- String msg = "Failed to write chunk " + chunkInfo.getChunkName() +
- " " + "into block " + blockID;
- LOG.debug("{}, exception: {}", msg, e.getLocalizedMessage());
- CompletionException ce = new CompletionException(msg, e);
- setIoException(ce);
- throw ce;
- }
- }, responseExecutor);
+ out.writeAsync(buf.nioBuffer(), StandardWriteOption.SYNC) :
+ out.writeAsync(buf.nioBuffer()))
+ .whenCompleteAsync((r, e) -> {
+ if (e != null || !r.isSuccess()) {
+ if (e == null) {
+ e = new IOException("result is not success");
+ }
+ String msg =
+ "Failed to write chunk " + chunkInfo.getChunkName() +
+ " " + "into block " + blockID;
+ LOG.debug("{}, exception: {}", msg, e.getLocalizedMessage());
+ CompletionException ce = new CompletionException(msg, e);
+ setIoException(ce);
+ throw ce;
+ }
+ }, responseExecutor);
futures.add(future);
containerBlockData.addChunks(chunkInfo);
@@ -754,7 +551,6 @@ public class BlockDataStreamOutput implements ByteBufStreamOutput {
*/
private void handleExecutionException(Exception ex) throws IOException {
setIoException(ex);
- adjustBuffersOnException();
throw getIoException();
}
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
new file mode 100644
index 0000000000..c187ffe902
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This class maintains the map of the commitIndexes to be watched for
+ * successful replication in the datanodes in a given pipeline. It also releases
+ * the buffers associated with the user data back to {@Link BufferPool} once
+ * minimum replication criteria is achieved during an ozone key write.
+ */
+package org.apache.hadoop.hdds.scm.storage;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.scm.XceiverClientReply;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * This class executes watchForCommit on ratis pipeline and releases
+ * buffers once data successfully gets replicated.
+ */
+public class StreamCommitWatcher {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StreamCommitWatcher.class);
+
+ private Set<Long> commitIndexSet;
+
+ // future Map to hold up all putBlock futures
+ private ConcurrentHashMap<Long,
+ CompletableFuture<ContainerCommandResponseProto>>
+ futureMap;
+
+ private XceiverClientSpi xceiverClient;
+
+ public StreamCommitWatcher(XceiverClientSpi xceiverClient) {
+ this.xceiverClient = xceiverClient;
+ commitIndexSet = new ConcurrentSkipListSet();
+ futureMap = new ConcurrentHashMap<>();
+ }
+
+ public void updateCommitInfoSet(long index) {
+ commitIndexSet.add(index);
+ }
+
+ int getCommitInfoSetSize() {
+ return commitIndexSet.size();
+ }
+
+ /**
+ * Calls watch for commit for the first index in commitIndex2flushedDataMap to
+ * the Ratis client.
+ * @return {@link XceiverClientReply} reply from raft client
+ * @throws IOException in case watchForCommit fails
+ */
+ public XceiverClientReply streamWatchOnFirstIndex() throws IOException {
+ if (!commitIndexSet.isEmpty()) {
+ // wait for the first commit index in the commitIndex2flushedDataMap
+ // to get committed to all or majority of nodes in case timeout
+ // happens.
+ long index =
+ commitIndexSet.stream().mapToLong(v -> v).min()
+ .getAsLong();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("waiting for first index {} to catch up", index);
+ }
+ return streamWatchForCommit(index);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Calls watch for commit for the last index in commitIndex2flushedDataMap to
+ * the Ratis client.
+ * @return {@link XceiverClientReply} reply from raft client
+ * @throws IOException in case watchForCommit fails
+ */
+ public XceiverClientReply streamWatchOnLastIndex()
+ throws IOException {
+ if (!commitIndexSet.isEmpty()) {
+ // wait for the commit index in the commitIndex2flushedDataMap
+ // to get committed to all or majority of nodes in case timeout
+ // happens.
+ long index =
+ commitIndexSet.stream().mapToLong(v -> v).max()
+ .getAsLong();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("waiting for last flush Index {} to catch up", index);
+ }
+ return streamWatchForCommit(index);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * calls watchForCommit API of the Ratis Client. This method is for streaming
+ * and no longer requires releaseBuffers
+ * @param commitIndex log index to watch for
+ * @return minimum commit index replicated to all nodes
+ * @throws IOException IOException in case watch gets timed out
+ */
+ public XceiverClientReply streamWatchForCommit(long commitIndex)
+ throws IOException {
+ try {
+ XceiverClientReply reply =
+ xceiverClient.watchForCommit(commitIndex);
+ return reply;
+ } catch (InterruptedException e) {
+ // Re-interrupt the thread while catching InterruptedException
+ Thread.currentThread().interrupt();
+ throw getIOExceptionForWatchForCommit(commitIndex, e);
+ } catch (TimeoutException | ExecutionException e) {
+ throw getIOExceptionForWatchForCommit(commitIndex, e);
+ }
+ }
+
+ private IOException getIOExceptionForWatchForCommit(long commitIndex,
+ Exception e) {
+ LOG.warn("watchForCommit failed for index {}", commitIndex, e);
+ IOException ioException = new IOException(
+ "Unexpected Storage Container Exception: " + e.toString(), e);
+ return ioException;
+ }
+
+ public ConcurrentMap<Long,
+ CompletableFuture<
+ ContainerCommandResponseProto>> getFutureMap() {
+ return futureMap;
+ }
+
+ public void cleanup() {
+ if (commitIndexSet != null) {
+ commitIndexSet.clear();
+ }
+ if (futureMap != null) {
+ futureMap.clear();
+ }
+ commitIndexSet = null;
+ }
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
index 6954742601..98907bf8af 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
-import org.apache.hadoop.hdds.scm.storage.BufferPool;
import org.apache.hadoop.hdds.scm.storage.ByteBufStreamOutput;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.security.token.Token;
@@ -52,15 +51,12 @@ public final class BlockDataStreamOutputEntry
private long currentPosition;
private final Token<OzoneBlockTokenIdentifier> token;
- private BufferPool bufferPool;
-
@SuppressWarnings({"parameternumber", "squid:S00107"})
private BlockDataStreamOutputEntry(
BlockID blockID, String key,
XceiverClientFactory xceiverClientManager,
Pipeline pipeline,
long length,
- BufferPool bufferPool,
Token<OzoneBlockTokenIdentifier> token,
OzoneClientConfig config
) {
@@ -73,7 +69,6 @@ public final class BlockDataStreamOutputEntry
this.token = token;
this.length = length;
this.currentPosition = 0;
- this.bufferPool = bufferPool;
}
long getLength() {
@@ -98,7 +93,7 @@ public final class BlockDataStreamOutputEntry
if (this.byteBufStreamOutput == null) {
this.byteBufStreamOutput =
new BlockDataStreamOutput(blockID, xceiverClientManager,
- pipeline, bufferPool, config, token);
+ pipeline, config, token);
}
}
@@ -135,20 +130,6 @@ public final class BlockDataStreamOutputEntry
return false;
}
- long getTotalAckDataLength() {
- if (byteBufStreamOutput != null) {
- BlockDataStreamOutput out =
- (BlockDataStreamOutput) this.byteBufStreamOutput;
- blockID = out.getBlockID();
- return out.getTotalAckDataLength();
- } else {
- // For a pre allocated block for which no write has been initiated,
- // the ByteBufStreamOutput will be null here.
- // In such cases, the default blockCommitSequenceId will be 0
- return 0;
- }
- }
-
Collection<DatanodeDetails> getFailedServers() {
if (byteBufStreamOutput != null) {
BlockDataStreamOutput out =
@@ -198,7 +179,6 @@ public final class BlockDataStreamOutputEntry
private XceiverClientFactory xceiverClientManager;
private Pipeline pipeline;
private long length;
- private BufferPool bufferPool;
private Token<OzoneBlockTokenIdentifier> token;
private OzoneClientConfig config;
@@ -230,12 +210,6 @@ public final class BlockDataStreamOutputEntry
return this;
}
-
- public Builder setBufferPool(BufferPool pool) {
- this.bufferPool = pool;
- return this;
- }
-
public Builder setConfig(OzoneClientConfig clientConfig) {
this.config = clientConfig;
return this;
@@ -252,7 +226,6 @@ public final class BlockDataStreamOutputEntry
xceiverClientManager,
pipeline,
length,
- bufferPool,
token, config);
}
}
@@ -282,10 +255,6 @@ public final class BlockDataStreamOutputEntry
return currentPosition;
}
- public BufferPool getBufferPool() {
- return bufferPool;
- }
-
public void setCurrentPosition(long curPosition) {
this.currentPosition = curPosition;
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index 94c505f2af..4bc55de262 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -22,12 +22,10 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.ByteStringConversion;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.storage.BufferPool;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -58,7 +56,6 @@ public class BlockDataStreamOutputEntryPool {
private final OmKeyArgs keyArgs;
private final XceiverClientFactory xceiverClientFactory;
private final String requestID;
- private final BufferPool bufferPool;
private OmMultipartCommitUploadPartInfo commitUploadPartInfo;
private final long openID;
private final ExcludeList excludeList;
@@ -86,13 +83,6 @@ public class BlockDataStreamOutputEntryPool {
this.requestID = requestId;
this.openID = openID;
this.excludeList = new ExcludeList();
-
- this.bufferPool =
- new BufferPool(config.getStreamBufferSize(),
- (int) (config.getStreamBufferMaxSize() / config
- .getStreamBufferSize()),
- ByteStringConversion
- .createByteBufferConversion(unsafeByteBufferConversion));
}
/**
@@ -114,8 +104,6 @@ public class BlockDataStreamOutputEntryPool {
config.setStreamBufferFlushDelay(false);
requestID = null;
int chunkSize = 0;
- bufferPool = new BufferPool(chunkSize, 1);
-
currentStreamIndex = 0;
openID = -1;
excludeList = new ExcludeList();
@@ -154,7 +142,6 @@ public class BlockDataStreamOutputEntryPool {
.setPipeline(subKeyInfo.getPipeline())
.setConfig(config)
.setLength(subKeyInfo.getLength())
- .setBufferPool(bufferPool)
.setToken(subKeyInfo.getToken());
streamEntries.add(builder.build());
}
@@ -293,17 +280,10 @@ public class BlockDataStreamOutputEntryPool {
return streamEntries.get(currentStreamIndex);
}
- long computeBufferData() {
- return bufferPool.computeBufferData();
- }
-
void cleanup() {
if (excludeList != null) {
excludeList.clear();
}
- if (bufferPool != null) {
- bufferPool.clearBufferPool();
- }
if (streamEntries != null) {
streamEntries.clear();
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
index a9be11667c..c37f9cd51d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
@@ -279,27 +279,7 @@ public class KeyDataStreamOutput implements ByteBufStreamOutput {
}
Pipeline pipeline = streamEntry.getPipeline();
PipelineID pipelineId = pipeline.getId();
- long totalSuccessfulFlushedData = streamEntry.getTotalAckDataLength();
- //set the correct length for the current stream
- streamEntry.setCurrentPosition(totalSuccessfulFlushedData);
- long bufferedDataLen = blockDataStreamOutputEntryPool.computeBufferData();
- if (containerExclusionException) {
- LOG.debug(
- "Encountered exception {}. The last committed block length is {}, "
- + "uncommitted data length is {} retry count {}", exception,
- totalSuccessfulFlushedData, bufferedDataLen, retryCount);
- } else {
- LOG.warn(
- "Encountered exception {} on the pipeline {}. "
- + "The last committed block length is {}, "
- + "uncommitted data length is {} retry count {}", exception,
- pipeline, totalSuccessfulFlushedData, bufferedDataLen, retryCount);
- }
- Preconditions.checkArgument(
- bufferedDataLen <= config.getStreamBufferMaxSize());
- Preconditions.checkArgument(
- offset - blockDataStreamOutputEntryPool.getKeyLength() ==
- bufferedDataLen);
+
long containerId = streamEntry.getBlockID().getContainerID();
Collection<DatanodeDetails> failedServers = streamEntry.getFailedServers();
Preconditions.checkNotNull(failedServers);
@@ -337,13 +317,6 @@ public class KeyDataStreamOutput implements ByteBufStreamOutput {
blockDataStreamOutputEntryPool
.discardPreallocatedBlocks(-1, pipelineId);
}
- if (bufferedDataLen > 0) {
- // If the data is still cached in the underlying stream, we need to
- // allocate new block and write this data in the datanode.
- handleRetry(exception, bufferedDataLen);
- // reset the retryCount after handling the exception
- retryCount = 0;
- }
}
private void markStreamClosed() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 12/35: HDDS-5895. [Ozone-Streaming] Make raft.server.data-stream.client.pool.size configurable (#2766)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2c1365e1b47c628d2f250f8b8db3a4872e1e374b
Author: micah zhao <mi...@tencent.com>
AuthorDate: Tue Oct 26 15:10:58 2021 +0800
HDDS-5895. [Ozone-Streaming] Make raft.server.data-stream.client.pool.size configurable (#2766)
---
.../transport/server/ratis/XceiverServerRatis.java | 5 +++++
.../hadoop/hdds/conf/DatanodeRatisServerConfig.java | 17 +++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index d69b64cce1..2fcc07fc23 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -242,6 +242,11 @@ public final class XceiverServerRatis implements XceiverServerSpi {
.getStreamWriteThreads();
RaftServerConfigKeys.DataStream.setAsyncWriteThreadPoolSize(properties,
dataStreamWriteRequestThreadPoolSize);
+ int dataStreamClientPoolSize =
+ conf.getObject(DatanodeRatisServerConfig.class)
+ .getClientPoolSize();
+ RaftServerConfigKeys.DataStream.setClientPoolSize(properties,
+ dataStreamClientPoolSize);
}
@SuppressWarnings("checkstyle:methodlength")
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
index 205d92e955..3132928abe 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
@@ -158,6 +158,23 @@ public class DatanodeRatisServerConfig {
this.streamWriteThreads = streamWriteThreads;
}
+ @Config(key = "datastream.client.pool.size",
+ defaultValue = "10",
+ type = ConfigType.INT,
+ tags = {OZONE, DATANODE, RATIS, DATASTREAM},
+ description = "Maximum number of client proxy in NettyServerStreamRpc " +
+ "for datastream write."
+ )
+ private int clientPoolSize;
+
+ public int getClientPoolSize() {
+ return clientPoolSize;
+ }
+
+ public void setClientPoolSize(int clientPoolSize) {
+ this.clientPoolSize = clientPoolSize;
+ }
+
@Config(key = "delete.ratis.log.directory",
defaultValue = "true",
type = ConfigType.BOOLEAN,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 13/35: HDDS-5763. Provide an Executor for each LocalStream in ContainerStateMachine (#2782)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0f5a55a62dd90831b713c013cd7ac0d988c50426
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Mon Nov 1 23:39:53 2021 +0800
HDDS-5763. Provide an Executor for each LocalStream in ContainerStateMachine (#2782)
---
.../transport/server/ratis/ContainerStateMachine.java | 7 +++++--
.../common/transport/server/ratis/LocalStream.java | 10 +++++++++-
.../transport/server/ratis/XceiverServerRatis.java | 5 -----
.../hadoop/hdds/conf/DatanodeRatisServerConfig.java | 17 -----------------
4 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index aed81ab5d8..ed5659d1d7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -527,8 +527,11 @@ public class ContainerStateMachine extends BaseStateMachine {
ContainerCommandResponseProto response = runCommand(
requestProto, context);
- String path = response.getMessage();
- return new LocalStream(new StreamDataChannel(Paths.get(path)));
+ final StreamDataChannel channel = new StreamDataChannel(
+ Paths.get(response.getMessage()));
+ final ExecutorService chunkExecutor = requestProto.hasWriteChunk() ?
+ getChunkExecutor(requestProto.getWriteChunk()) : null;
+ return new LocalStream(channel, chunkExecutor);
} catch (IOException e) {
throw new CompletionException("Failed to create data stream", e);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
index baae013966..780f874398 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
@@ -23,12 +23,15 @@ import org.apache.ratis.statemachine.StateMachine;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
class LocalStream implements StateMachine.DataStream {
private final StateMachine.DataChannel dataChannel;
+ private final Executor executor;
- LocalStream(StateMachine.DataChannel dataChannel) {
+ LocalStream(StateMachine.DataChannel dataChannel, Executor executor) {
this.dataChannel = dataChannel;
+ this.executor = executor;
}
@Override
@@ -47,4 +50,9 @@ class LocalStream implements StateMachine.DataStream {
}
});
}
+
+ @Override
+ public Executor getExecutor() {
+ return executor;
+ }
}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 2fcc07fc23..6b0ad0e41e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -237,11 +237,6 @@ public final class XceiverServerRatis implements XceiverServerSpi {
.getStreamRequestThreads();
RaftServerConfigKeys.DataStream.setAsyncRequestThreadPoolSize(properties,
dataStreamAsyncRequestThreadPoolSize);
- int dataStreamWriteRequestThreadPoolSize =
- conf.getObject(DatanodeRatisServerConfig.class)
- .getStreamWriteThreads();
- RaftServerConfigKeys.DataStream.setAsyncWriteThreadPoolSize(properties,
- dataStreamWriteRequestThreadPoolSize);
int dataStreamClientPoolSize =
conf.getObject(DatanodeRatisServerConfig.class)
.getClientPoolSize();
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
index 3132928abe..058932e769 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
@@ -141,23 +141,6 @@ public class DatanodeRatisServerConfig {
this.streamRequestThreads = streamRequestThreads;
}
- @Config(key = "datastream.write.threads",
- defaultValue = "20",
- type = ConfigType.INT,
- tags = {OZONE, DATANODE, RATIS, DATASTREAM},
- description = "Maximum number of threads in the thread pool for " +
- "datastream write."
- )
- private int streamWriteThreads;
-
- public int getStreamWriteThreads() {
- return streamWriteThreads;
- }
-
- public void setStreamWriteThreads(int streamWriteThreads) {
- this.streamWriteThreads = streamWriteThreads;
- }
-
@Config(key = "datastream.client.pool.size",
defaultValue = "10",
type = ConfigType.INT,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 22/35: HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 59eb086e9a9fbd6757a9e82ae74bdb9e23d75202
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Fri Jan 14 20:12:13 2022 +0530
HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
---
.../org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 6ef59dd6d8..ec925d1e6a 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -343,6 +343,10 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
while (len > 0) {
final StreamBuffer buf = bufferList.get(count);
final long writeLen = Math.min(buf.position(), len);
+ if (buffersForPutBlock == null) {
+ buffersForPutBlock = new ArrayList<>();
+ }
+ buffersForPutBlock.add(buf);
final ByteBuffer duplicated = buf.duplicate();
duplicated.position(0);
duplicated.limit(buf.position());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 11/35: HDDS-5674.[Ozone-Streaming] Handle client retries on exception (#2701)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 5049243ac1eb027a6664127aec9eb7b6bfe8e8d8
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Thu Oct 21 12:57:30 2021 +0530
HDDS-5674.[Ozone-Streaming] Handle client retries on exception (#2701)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 62 ++++++++++++---
.../hadoop/hdds/scm/storage/StreamBuffer.java | 46 +++++++++++
.../hdds/scm/storage/StreamCommitWatcher.java | 93 ++++++++++++++++++----
.../client/io/BlockDataStreamOutputEntry.java | 33 +++++++-
.../client/io/BlockDataStreamOutputEntryPool.java | 14 +++-
.../ozone/client/io/KeyDataStreamOutput.java | 12 ++-
.../client/rpc/TestBlockDataStreamOutput.java | 30 +++++++
.../apache/hadoop/ozone/container/TestHelper.java | 20 +++++
8 files changed, 279 insertions(+), 31 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 2ae0ba7525..aada48e2f5 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -92,6 +92,11 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
private int chunkIndex;
private final AtomicLong chunkOffset = new AtomicLong();
+
+ // Similar to 'BufferPool' but this list maintains only references
+ // to the ByteBuffers.
+ private List<StreamBuffer> bufferList;
+
// The IOException will be set by response handling thread in case there is an
// exception received in the response. If the exception is set, the next
// request will fail upfront.
@@ -133,7 +138,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
XceiverClientFactory xceiverClientManager,
Pipeline pipeline,
OzoneClientConfig config,
- Token<? extends TokenIdentifier> token
+ Token<? extends TokenIdentifier> token,
+ List<StreamBuffer> bufferList
) throws IOException {
this.xceiverClientFactory = xceiverClientManager;
this.config = config;
@@ -148,7 +154,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// Alternatively, stream setup can be delayed till the first chunk write.
this.out = setupStream(pipeline);
this.token = token;
-
+ this.bufferList = bufferList;
flushPeriod = (int) (config.getStreamBufferFlushSize() / config
.getStreamBufferSize());
@@ -159,7 +165,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// A single thread executor handle the responses of async requests
responseExecutor = Executors.newSingleThreadExecutor();
- commitWatcher = new StreamCommitWatcher(xceiverClient);
+ commitWatcher = new StreamCommitWatcher(xceiverClient, bufferList);
totalDataFlushedLength = 0;
writtenDataLength = 0;
failedServers = new ArrayList<>(0);
@@ -251,8 +257,11 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
if (len == 0) {
return;
}
- writeChunkToContainer(
- (ByteBuffer) b.asReadOnlyBuffer().position(off).limit(off + len));
+
+ final StreamBuffer buf = new StreamBuffer(b, off, len);
+ bufferList.add(buf);
+
+ writeChunkToContainer(buf.duplicate());
writtenDataLength += len;
}
@@ -261,6 +270,10 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
totalDataFlushedLength = writtenDataLength;
}
+ @VisibleForTesting
+ public long getTotalDataFlushedLength() {
+ return totalDataFlushedLength;
+ }
/**
* Will be called on the retryPath in case closedContainerException/
* TimeoutException.
@@ -268,8 +281,27 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
* @throws IOException if error occurred
*/
- // TODO: We need add new retry policy without depend on bufferPool.
public void writeOnRetry(long len) throws IOException {
+ if (len == 0) {
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrying write length {} for blockID {}", len, blockID);
+ }
+ int count = 0;
+ while (len > 0) {
+ final StreamBuffer buf = bufferList.get(count);
+ final long writeLen = Math.min(buf.length(), len);
+ final ByteBuffer duplicated = buf.duplicate();
+ if (writeLen != buf.length()) {
+ duplicated.limit(Math.toIntExact(len));
+ }
+ writeChunkToContainer(duplicated);
+ len -= writeLen;
+ count++;
+ writtenDataLength += writeLen;
+ }
+
}
@@ -314,6 +346,14 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
boolean force) throws IOException {
checkOpen();
long flushPos = totalDataFlushedLength;
+ final List<StreamBuffer> byteBufferList;
+ if (!force) {
+ Preconditions.checkNotNull(bufferList);
+ byteBufferList = bufferList;
+ Preconditions.checkNotNull(byteBufferList);
+ } else {
+ byteBufferList = null;
+ }
flush();
if (close) {
dataStreamCloseReply = out.closeAsync();
@@ -344,12 +384,12 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding index " + asyncReply.getLogIndex() + " commitMap size "
- + commitWatcher.getCommitInfoSetSize() + " flushLength "
+ + commitWatcher.getCommitInfoMapSize() + " flushLength "
+ flushPos + " blockID " + blockID);
}
// for standalone protocol, logIndex will always be 0.
- commitWatcher.updateCommitInfoSet(
- asyncReply.getLogIndex());
+ commitWatcher
+ .updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
}
return e;
}, responseExecutor).exceptionally(e -> {
@@ -589,4 +629,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
setIoException(ex);
throw getIoException();
}
+
+ public long getTotalAckDataLength() {
+ return commitWatcher.getTotalAckDataLength();
+ }
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
new file mode 100644
index 0000000000..f36019e2ae
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Used for streaming write.
+ */
+public class StreamBuffer {
+ private final ByteBuffer buffer;
+
+ public StreamBuffer(ByteBuffer buffer) {
+ this.buffer = buffer.asReadOnlyBuffer();
+ }
+
+ public StreamBuffer(ByteBuffer buffer, int offset, int length) {
+ this((ByteBuffer) buffer.asReadOnlyBuffer().position(offset)
+ .limit(offset + length));
+ }
+
+ public ByteBuffer duplicate() {
+ return buffer.duplicate();
+ }
+
+ public int length() {
+ return buffer.limit() - buffer.position();
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
index c187ffe902..3a59d07571 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
@@ -24,6 +24,7 @@
*/
package org.apache.hadoop.hdds.scm.storage;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
@@ -31,13 +32,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Set;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
/**
* This class executes watchForCommit on ratis pipeline and releases
@@ -48,7 +52,12 @@ public class StreamCommitWatcher {
private static final Logger LOG =
LoggerFactory.getLogger(StreamCommitWatcher.class);
- private Set<Long> commitIndexSet;
+ private Map<Long, List<StreamBuffer>> commitIndexMap;
+ private List<StreamBuffer> bufferList;
+
+ // total data which has been successfully flushed and acknowledged
+ // by all servers
+ private long totalAckDataLength;
// future Map to hold up all putBlock futures
private ConcurrentHashMap<Long,
@@ -57,18 +66,22 @@ public class StreamCommitWatcher {
private XceiverClientSpi xceiverClient;
- public StreamCommitWatcher(XceiverClientSpi xceiverClient) {
+ public StreamCommitWatcher(XceiverClientSpi xceiverClient,
+ List<StreamBuffer> bufferList) {
this.xceiverClient = xceiverClient;
- commitIndexSet = new ConcurrentSkipListSet();
+ commitIndexMap = new ConcurrentSkipListMap<>();
futureMap = new ConcurrentHashMap<>();
+ this.bufferList = bufferList;
+ totalAckDataLength = 0;
}
- public void updateCommitInfoSet(long index) {
- commitIndexSet.add(index);
+ public void updateCommitInfoMap(long index, List<StreamBuffer> buffers) {
+ commitIndexMap.computeIfAbsent(index, k -> new LinkedList<>())
+ .addAll(buffers);
}
- int getCommitInfoSetSize() {
- return commitIndexSet.size();
+ int getCommitInfoMapSize() {
+ return commitIndexMap.size();
}
/**
@@ -78,12 +91,12 @@ public class StreamCommitWatcher {
* @throws IOException in case watchForCommit fails
*/
public XceiverClientReply streamWatchOnFirstIndex() throws IOException {
- if (!commitIndexSet.isEmpty()) {
+ if (!commitIndexMap.isEmpty()) {
// wait for the first commit index in the commitIndex2flushedDataMap
// to get committed to all or majority of nodes in case timeout
// happens.
long index =
- commitIndexSet.stream().mapToLong(v -> v).min()
+ commitIndexMap.keySet().stream().mapToLong(v -> v).min()
.getAsLong();
if (LOG.isDebugEnabled()) {
LOG.debug("waiting for first index {} to catch up", index);
@@ -102,12 +115,12 @@ public class StreamCommitWatcher {
*/
public XceiverClientReply streamWatchOnLastIndex()
throws IOException {
- if (!commitIndexSet.isEmpty()) {
+ if (!commitIndexMap.isEmpty()) {
// wait for the commit index in the commitIndex2flushedDataMap
// to get committed to all or majority of nodes in case timeout
// happens.
long index =
- commitIndexSet.stream().mapToLong(v -> v).max()
+ commitIndexMap.keySet().stream().mapToLong(v -> v).max()
.getAsLong();
if (LOG.isDebugEnabled()) {
LOG.debug("waiting for last flush Index {} to catch up", index);
@@ -127,9 +140,16 @@ public class StreamCommitWatcher {
*/
public XceiverClientReply streamWatchForCommit(long commitIndex)
throws IOException {
+ final long index;
try {
XceiverClientReply reply =
xceiverClient.watchForCommit(commitIndex);
+ if (reply == null) {
+ index = 0;
+ } else {
+ index = reply.getLogIndex();
+ }
+ adjustBuffers(index);
return reply;
} catch (InterruptedException e) {
// Re-interrupt the thread while catching InterruptedException
@@ -140,11 +160,52 @@ public class StreamCommitWatcher {
}
}
+ void releaseBuffersOnException() {
+ adjustBuffers(xceiverClient.getReplicatedMinCommitIndex());
+ }
+
+ private void adjustBuffers(long commitIndex) {
+ List<Long> keyList = commitIndexMap.keySet().stream()
+ .filter(p -> p <= commitIndex).collect(Collectors.toList());
+ if (!keyList.isEmpty()) {
+ releaseBuffers(keyList);
+ }
+ }
+
+ private long releaseBuffers(List<Long> indexes) {
+ Preconditions.checkArgument(!commitIndexMap.isEmpty());
+ for (long index : indexes) {
+ Preconditions.checkState(commitIndexMap.containsKey(index));
+ final List<StreamBuffer> buffers = commitIndexMap.remove(index);
+ final long length =
+ buffers.stream().mapToLong(StreamBuffer::length).sum();
+ totalAckDataLength += length;
+ // clear the future object from the future Map
+ final CompletableFuture<ContainerCommandResponseProto> remove =
+ futureMap.remove(totalAckDataLength);
+ if (remove == null) {
+ LOG.error("Couldn't find required future for " + totalAckDataLength);
+ for (Long key : futureMap.keySet()) {
+ LOG.error("Existing acknowledged data: " + key);
+ }
+ }
+ for (StreamBuffer byteBuffer : buffers) {
+ bufferList.remove(byteBuffer);
+ }
+ }
+ return totalAckDataLength;
+ }
+
+ public long getTotalAckDataLength() {
+ return totalAckDataLength;
+ }
+
private IOException getIOExceptionForWatchForCommit(long commitIndex,
Exception e) {
LOG.warn("watchForCommit failed for index {}", commitIndex, e);
IOException ioException = new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
+ releaseBuffersOnException();
return ioException;
}
@@ -155,12 +216,12 @@ public class StreamCommitWatcher {
}
public void cleanup() {
- if (commitIndexSet != null) {
- commitIndexSet.clear();
+ if (commitIndexMap != null) {
+ commitIndexMap.clear();
}
if (futureMap != null) {
futureMap.clear();
}
- commitIndexSet = null;
+ commitIndexMap = null;
}
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
index f0c3a43e89..2cd5630549 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.StreamBuffer;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.security.token.Token;
@@ -32,6 +33,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
/**
* Helper class used inside {@link BlockDataStreamOutput}.
@@ -50,6 +52,7 @@ public final class BlockDataStreamOutputEntry
// the current position of this stream 0 <= currentPosition < length
private long currentPosition;
private final Token<OzoneBlockTokenIdentifier> token;
+ private List<StreamBuffer> bufferList;
@SuppressWarnings({"parameternumber", "squid:S00107"})
private BlockDataStreamOutputEntry(
@@ -58,7 +61,8 @@ public final class BlockDataStreamOutputEntry
Pipeline pipeline,
long length,
Token<OzoneBlockTokenIdentifier> token,
- OzoneClientConfig config
+ OzoneClientConfig config,
+ List<StreamBuffer> bufferList
) {
this.config = config;
this.byteBufferStreamOutput = null;
@@ -69,6 +73,7 @@ public final class BlockDataStreamOutputEntry
this.token = token;
this.length = length;
this.currentPosition = 0;
+ this.bufferList = bufferList;
}
long getLength() {
@@ -92,8 +97,8 @@ public final class BlockDataStreamOutputEntry
private void checkStream() throws IOException {
if (this.byteBufferStreamOutput == null) {
this.byteBufferStreamOutput =
- new BlockDataStreamOutput(blockID, xceiverClientManager,
- pipeline, config, token);
+ new BlockDataStreamOutput(blockID, xceiverClientManager, pipeline,
+ config, token, bufferList);
}
}
@@ -151,6 +156,20 @@ public final class BlockDataStreamOutputEntry
}
}
+ long getTotalAckDataLength() {
+ if (byteBufferStreamOutput != null) {
+ BlockDataStreamOutput out =
+ (BlockDataStreamOutput) this.byteBufferStreamOutput;
+ blockID = out.getBlockID();
+ return out.getTotalAckDataLength();
+ } else {
+ // For a pre allocated block for which no write has been initiated,
+ // the OutputStream will be null here.
+ // In such cases, the default blockCommitSequenceId will be 0
+ return 0;
+ }
+ }
+
void cleanup(boolean invalidateClient) throws IOException {
checkStream();
BlockDataStreamOutput out =
@@ -180,6 +199,7 @@ public final class BlockDataStreamOutputEntry
private long length;
private Token<OzoneBlockTokenIdentifier> token;
private OzoneClientConfig config;
+ private List<StreamBuffer> bufferList;
public Builder setBlockID(BlockID bID) {
this.blockID = bID;
@@ -219,13 +239,18 @@ public final class BlockDataStreamOutputEntry
return this;
}
+ public Builder setBufferList(List<StreamBuffer> bList) {
+ this.bufferList = bList;
+ return this;
+ }
+
public BlockDataStreamOutputEntry build() {
return new BlockDataStreamOutputEntry(blockID,
key,
xceiverClientManager,
pipeline,
length,
- token, config);
+ token, config, bufferList);
}
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index 4bc55de262..e49b0b79ad 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.StreamBuffer;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -59,6 +60,7 @@ public class BlockDataStreamOutputEntryPool {
private OmMultipartCommitUploadPartInfo commitUploadPartInfo;
private final long openID;
private final ExcludeList excludeList;
+ private List<StreamBuffer> bufferList;
@SuppressWarnings({"parameternumber", "squid:S00107"})
public BlockDataStreamOutputEntryPool(
@@ -83,6 +85,7 @@ public class BlockDataStreamOutputEntryPool {
this.requestID = requestId;
this.openID = openID;
this.excludeList = new ExcludeList();
+ this.bufferList = new ArrayList<>();
}
/**
@@ -142,7 +145,8 @@ public class BlockDataStreamOutputEntryPool {
.setPipeline(subKeyInfo.getPipeline())
.setConfig(config)
.setLength(subKeyInfo.getLength())
- .setToken(subKeyInfo.getToken());
+ .setToken(subKeyInfo.getToken())
+ .setBufferList(bufferList);
streamEntries.add(builder.build());
}
@@ -301,4 +305,12 @@ public class BlockDataStreamOutputEntryPool {
boolean isEmpty() {
return streamEntries.isEmpty();
}
+
+ long computeBufferData() {
+ long totalDataLen =0;
+ for (StreamBuffer b : bufferList){
+ totalDataLen += b.length();
+ }
+ return totalDataLen;
+ }
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
index 9bba89d0a8..2540e42e24 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
@@ -278,11 +278,14 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
}
Pipeline pipeline = streamEntry.getPipeline();
PipelineID pipelineId = pipeline.getId();
-
+ long totalSuccessfulFlushedData = streamEntry.getTotalAckDataLength();
+ //set the correct length for the current stream
+ streamEntry.setCurrentPosition(totalSuccessfulFlushedData);
long containerId = streamEntry.getBlockID().getContainerID();
Collection<DatanodeDetails> failedServers = streamEntry.getFailedServers();
Preconditions.checkNotNull(failedServers);
ExcludeList excludeList = blockDataStreamOutputEntryPool.getExcludeList();
+ long bufferedDataLen = blockDataStreamOutputEntryPool.computeBufferData();
if (!failedServers.isEmpty()) {
excludeList.addDatanodes(failedServers);
}
@@ -316,6 +319,13 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
blockDataStreamOutputEntryPool
.discardPreallocatedBlocks(-1, pipelineId);
}
+ if (bufferedDataLen > 0) {
+ // If the data is still cached in the underlying stream, we need to
+ // allocate new block and write this data in the datanode.
+ handleRetry(exception, bufferedDataLen);
+ // reset the retryCount after handling the exception
+ retryCount = 0;
+ }
}
private void markStreamClosed() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index d3b2d22577..05a101951b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -21,15 +21,19 @@ import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
+import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.TestHelper;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -127,21 +131,25 @@ public class TestBlockDataStreamOutput {
@Test
public void testHalfChunkWrite() throws Exception {
testWrite(chunkSize / 2);
+ testWriteWithFailure(chunkSize/2);
}
@Test
public void testSingleChunkWrite() throws Exception {
testWrite(chunkSize);
+ testWriteWithFailure(chunkSize);
}
@Test
public void testMultiChunkWrite() throws Exception {
testWrite(chunkSize + 50);
+ testWriteWithFailure(chunkSize + 50);
}
@Test
public void testMultiBlockWrite() throws Exception {
testWrite(blockSize + 50);
+ testWriteWithFailure(blockSize + 50);
}
private void testWrite(int dataLength) throws Exception {
@@ -156,6 +164,28 @@ public class TestBlockDataStreamOutput {
key.close();
validateData(keyName, data);
}
+
+ private void testWriteWithFailure(int dataLength) throws Exception {
+ String keyName = getKeyName();
+ OzoneDataStreamOutput key = createKey(
+ keyName, ReplicationType.RATIS, 0);
+ byte[] data =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength)
+ .getBytes(UTF_8);
+ ByteBuffer b = ByteBuffer.wrap(data);
+ key.write(b);
+ KeyDataStreamOutput keyDataStreamOutput =
+ (KeyDataStreamOutput) key.getByteBufStreamOutput();
+ ByteBufferStreamOutput stream =
+ keyDataStreamOutput.getStreamEntries().get(0).getByteBufStreamOutput();
+ Assert.assertTrue(stream instanceof BlockDataStreamOutput);
+ TestHelper.waitForContainerClose(key, cluster);
+ key.write(b);
+ key.close();
+ String dataString = new String(data, UTF_8);
+ validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
+ }
+
private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
long size) throws Exception {
return TestHelper.createStreamKey(
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
index cf3a51241e..14cd1b66f4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java
@@ -46,7 +46,9 @@ import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.io.BlockDataStreamOutputEntry;
import org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry;
+import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
@@ -205,6 +207,24 @@ public final class TestHelper {
waitForContainerClose(cluster, containerIdList.toArray(new Long[0]));
}
+
+ public static void waitForContainerClose(OzoneDataStreamOutput outputStream,
+ MiniOzoneCluster cluster) throws Exception {
+ KeyDataStreamOutput keyOutputStream =
+ (KeyDataStreamOutput) outputStream.getByteBufStreamOutput();
+ List<BlockDataStreamOutputEntry> streamEntryList =
+ keyOutputStream.getStreamEntries();
+ List<Long> containerIdList = new ArrayList<>();
+ for (BlockDataStreamOutputEntry entry : streamEntryList) {
+ long id = entry.getBlockID().getContainerID();
+ if (!containerIdList.contains(id)) {
+ containerIdList.add(id);
+ }
+ }
+ Assert.assertTrue(!containerIdList.isEmpty());
+ waitForContainerClose(cluster, containerIdList.toArray(new Long[0]));
+ }
+
public static void waitForPipelineClose(OzoneOutputStream outputStream,
MiniOzoneCluster cluster, boolean waitForContainerCreation)
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 14/35: HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream config (#2841)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2a629f82c9a24db1e85537a7a95989163c0fd8d7
Author: hao guo <gu...@360.cn>
AuthorDate: Mon Nov 15 23:54:36 2021 +0800
HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream config (#2841)
---
.../src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index e93eb951ee..604aaf4b93 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -47,6 +47,7 @@ import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.datastream.SupportedDataStreamType;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.grpc.GrpcTlsConfig;
+import org.apache.ratis.netty.NettyConfigKeys;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
@@ -302,7 +303,8 @@ public final class RatisHelper {
Map<String, String> ratisClientConf =
getDatanodeRatisPrefixProps(ozoneConf);
ratisClientConf.forEach((key, val) -> {
- if (isClientConfig(key) || isGrpcClientConfig(key)) {
+ if (isClientConfig(key) || isGrpcClientConfig(key)
+ || isNettyStreamConfig(key)) {
raftProperties.set(key, val);
}
});
@@ -318,6 +320,11 @@ public final class RatisHelper {
!key.startsWith(GrpcConfigKeys.Admin.PREFIX) &&
!key.startsWith(GrpcConfigKeys.Server.PREFIX);
}
+
+ private static boolean isNettyStreamConfig(String key) {
+ return key.startsWith(NettyConfigKeys.DataStream.PREFIX);
+ }
+
/**
* Set all server properties matching with prefix
* {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 18/35: HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. (#2866)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4771a50e6adc10e3b0d388e78d9e4e33579c7afb
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Wed Dec 1 11:00:46 2021 +0530
HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer fixed boundary for streaming writes. (#2866)
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 31 ++++++++++++++++++++
.../hdds/scm/storage/BlockDataStreamOutput.java | 29 ++++++++++++++----
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 12 ++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 11 +++++++
.../client/rpc/TestBlockDataStreamOutput.java | 34 ++++++++++++++++++++++
5 files changed, 111 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index 63dd511596..4fad9df250 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -66,6 +66,21 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private int streamBufferSize = 4 * 1024 * 1024;
+ @Config(key = "datastream.max.buffer.size",
+ defaultValue = "4MB",
+ type = ConfigType.SIZE,
+ description = "The maximum size of the ByteBuffer "
+ + "(used via ratis streaming)",
+ tags = ConfigTag.CLIENT)
+ private int dataStreamMaxBufferSize = 4 * 1024 * 1024;
+
+ @Config(key = "datastream.buffer.flush.size",
+ defaultValue = "16MB",
+ type = ConfigType.SIZE,
+ description = "The boundary at which putBlock is executed",
+ tags = ConfigTag.CLIENT)
+ private long dataStreamBufferFlushSize = 16 * 1024 * 1024;
+
@Config(key = "stream.buffer.increment",
defaultValue = "0B",
type = ConfigType.SIZE,
@@ -220,6 +235,14 @@ public class OzoneClientConfig {
this.streamBufferSize = streamBufferSize;
}
+ public int getDataStreamMaxBufferSize() {
+ return dataStreamMaxBufferSize;
+ }
+
+ public void setDataStreamMaxBufferSize(int dataStreamMaxBufferSize) {
+ this.dataStreamMaxBufferSize = dataStreamMaxBufferSize;
+ }
+
public boolean isStreamBufferFlushDelay() {
return streamBufferFlushDelay;
}
@@ -288,6 +311,14 @@ public class OzoneClientConfig {
return bufferIncrement;
}
+ public long getDataStreamBufferFlushSize() {
+ return dataStreamBufferFlushSize;
+ }
+
+ public void setDataStreamBufferFlushSize(long dataStreamBufferFlushSize) {
+ this.dataStreamBufferFlushSize = dataStreamBufferFlushSize;
+ }
+
public ChecksumCombineMode getChecksumCombineMode() {
try {
return ChecksumCombineMode.valueOf(checksumCombineMode);
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index aada48e2f5..6f5a54354a 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -257,13 +257,30 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
if (len == 0) {
return;
}
+ int curLen = len;
+ // set limit on the number of bytes that a ByteBuffer(StreamBuffer) can hold
+ int maxBufferLen = config.getDataStreamMaxBufferSize();
+ while (curLen > 0) {
+ int writeLen = Math.min(curLen, maxBufferLen);
+ final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
+ off += writeLen;
+ bufferList.add(buf);
+ writeChunkToContainer(buf.duplicate());
+ curLen -= writeLen;
+ writtenDataLength += writeLen;
+ doFlushIfNeeded();
+ }
+ }
- final StreamBuffer buf = new StreamBuffer(b, off, len);
- bufferList.add(buf);
-
- writeChunkToContainer(buf.duplicate());
-
- writtenDataLength += len;
+ private void doFlushIfNeeded() throws IOException {
+ Preconditions.checkArgument(config.getDataStreamBufferFlushSize() > config
+ .getDataStreamMaxBufferSize());
+ long boundary = config.getDataStreamBufferFlushSize() / config
+ .getDataStreamMaxBufferSize();
+ if (bufferList.size() % boundary == 0) {
+ updateFlushLength();
+ executePutBlock(false, false);
+ }
}
private void updateFlushLength() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index b1d9a74209..501f9ce0fb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -323,6 +323,8 @@ public interface MiniOzoneCluster {
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferFlushSize = Optional.empty();
+ protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
+ protected OptionalInt dataStreamMaxBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferMaxSize = Optional.empty();
protected Optional<Long> blockSize = Optional.empty();
protected Optional<StorageUnit> streamBufferSizeUnit = Optional.empty();
@@ -557,6 +559,16 @@ public interface MiniOzoneCluster {
return this;
}
+ public Builder setDataStreamBufferMaxSize(int size) {
+ dataStreamMaxBufferSize = OptionalInt.of(size);
+ return this;
+ }
+
+ public Builder setDataStreamBufferFlushize(long size) {
+ dataStreamBufferFlushSize = Optional.of(size);
+ return this;
+ }
+
/**
* Sets the block size for stream buffer.
*
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index a549767481..8f75659ce9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -661,6 +661,12 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!streamBufferMaxSize.isPresent()) {
streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get());
}
+ if (!dataStreamBufferFlushSize.isPresent()) {
+ dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
+ }
+ if (!dataStreamMaxBufferSize.isPresent()) {
+ dataStreamMaxBufferSize = OptionalInt.of(chunkSize.get());
+ }
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
@@ -677,6 +683,11 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get())));
clientConfig.setStreamBufferFlushSize(Math.round(
streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get())));
+ clientConfig.setDataStreamBufferFlushSize(Math.round(
+ streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get())));
+ clientConfig.setDataStreamMaxBufferSize((int) Math.round(
+ streamBufferSizeUnit.get()
+ .toBytes(dataStreamMaxBufferSize.getAsInt())));
conf.setFromObject(clientConfig);
conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 05a101951b..5eb38a00de 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.ozone.client.rpc;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput;
import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -101,6 +104,8 @@ public class TestBlockDataStreamOutput {
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
.setStreamBufferMaxSize(maxFlushSize)
+ .setDataStreamBufferFlushize(maxFlushSize)
+ .setDataStreamBufferMaxSize(chunkSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
.build();
cluster.waitForClusterToBeReady();
@@ -186,6 +191,35 @@ public class TestBlockDataStreamOutput {
validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}
+ @Test
+ public void testPutBlockAtBoundary() throws Exception {
+ int dataLength = 500;
+ XceiverClientMetrics metrics =
+ XceiverClientManager.getXceiverClientMetrics();
+ long putBlockCount = metrics.getContainerOpCountMetrics(
+ ContainerProtos.Type.PutBlock);
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
+ ContainerProtos.Type.PutBlock);
+ String keyName = getKeyName();
+ OzoneDataStreamOutput key = createKey(
+ keyName, ReplicationType.RATIS, 0);
+ byte[] data =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength)
+ .getBytes(UTF_8);
+ key.write(ByteBuffer.wrap(data));
+ Assert.assertTrue(
+ metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
+ <= pendingPutBlockCount + 1);
+ key.close();
+ // Since data length is 500 , first putBlock will be at 400(flush boundary)
+ // and the other at 500
+ Assert.assertTrue(
+ metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
+ == putBlockCount + 2);
+ validateData(keyName, data);
+ }
+
+
private OzoneDataStreamOutput createKey(String keyName, ReplicationType type,
long size) throws Exception {
return TestHelper.createStreamKey(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 10/35: HDDS-5849. [Ozone-Streaming]Write exceptions occur after checksum is enabled (#2729)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 247c4f232454c7b01d1b37dc764a0791c268a0d5
Author: micah zhao <mi...@tencent.com>
AuthorDate: Tue Oct 12 14:05:07 2021 +0800
HDDS-5849. [Ozone-Streaming]Write exceptions occur after checksum is enabled (#2729)
---
.../java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 3 ++-
.../org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java | 2 --
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 41e2c48bbb..2ae0ba7525 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -519,7 +519,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
throws IOException {
final int effectiveChunkSize = buf.remaining();
final long offset = chunkOffset.getAndAdd(effectiveChunkSize);
- ChecksumData checksumData = checksum.computeChecksum(buf);
+ ChecksumData checksumData = checksum.computeChecksum(
+ buf.asReadOnlyBuffer());
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(blockID.get().getLocalID() + "_chunk_" + ++chunkIndex)
.setOffset(offset)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 6d5401d651..d3b2d22577 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.client.rpc;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -82,7 +81,6 @@ public class TestBlockDataStreamOutput {
blockSize = 2 * maxFlushSize;
OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setChecksumType(ChecksumType.NONE);
clientConfig.setStreamBufferFlushDelay(false);
conf.setFromObject(clientConfig);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 33/35: HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput. (#3195)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit b23fbcf9bd9d812912e154a93c726ed134914c84
Author: hao guo <gu...@360.cn>
AuthorDate: Mon Mar 28 16:16:31 2022 +0800
HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput. (#3195)
---
.../hdds/scm/storage/AbstractDataStreamOutput.java | 130 +++++++++++++++++++++
.../hdds/scm/storage/BlockDataStreamOutput.java | 36 +-----
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 32 +++++
.../ozone/container/ContainerTestHelper.java | 12 ++
.../server/ratis/ContainerStateMachine.java | 35 ++++--
.../ozone/container/keyvalue/KeyValueHandler.java | 33 +++---
.../keyvalue/impl/KeyValueStreamDataChannel.java | 2 +-
.../client/io/BlockDataStreamOutputEntryPool.java | 26 -----
.../ozone/client/io/KeyDataStreamOutput.java | 121 ++-----------------
.../client/rpc/TestBlockDataStreamOutput.java | 4 +-
.../rpc/TestContainerStateMachineStream.java | 59 ++++++++--
11 files changed, 279 insertions(+), 211 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/AbstractDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/AbstractDataStreamOutput.java
new file mode 100644
index 0000000000..e29670d781
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/AbstractDataStreamOutput.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.ratis.protocol.exceptions.AlreadyClosedException;
+import org.apache.ratis.protocol.exceptions.RaftRetryFailureException;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.Map;
+
+/**
+ * This class is used for error handling methods.
+ */
+public abstract class AbstractDataStreamOutput
+ implements ByteBufferStreamOutput {
+
+ private final Map<Class<? extends Throwable>, RetryPolicy> retryPolicyMap;
+ private int retryCount;
+ private boolean isException;
+
+ protected AbstractDataStreamOutput(
+ Map<Class<? extends Throwable>, RetryPolicy> retryPolicyMap) {
+ this.retryPolicyMap = retryPolicyMap;
+ this.isException = false;
+ this.retryCount = 0;
+ }
+
+ @VisibleForTesting
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ protected void resetRetryCount() {
+ retryCount = 0;
+ }
+
+ protected boolean isException() {
+ return isException;
+ }
+
+ /**
+ * Checks if the provided exception signifies retry failure in ratis client.
+ * In case of retry failure, ratis client throws RaftRetryFailureException
+ * and all succeeding operations are failed with AlreadyClosedException.
+ */
+ protected boolean checkForRetryFailure(Throwable t) {
+ return t instanceof RaftRetryFailureException
+ || t instanceof AlreadyClosedException;
+ }
+
+ // Every container specific exception from datatnode will be seen as
+ // StorageContainerException
+ protected boolean checkIfContainerToExclude(Throwable t) {
+ return t instanceof StorageContainerException;
+ }
+
+ protected void setExceptionAndThrow(IOException ioe) throws IOException {
+ isException = true;
+ throw ioe;
+ }
+
+ protected void handleRetry(IOException exception) throws IOException {
+ RetryPolicy retryPolicy = retryPolicyMap
+ .get(HddsClientUtils.checkForException(exception).getClass());
+ if (retryPolicy == null) {
+ retryPolicy = retryPolicyMap.get(Exception.class);
+ }
+ handleRetry(exception, retryPolicy);
+ }
+
+ protected void handleRetry(IOException exception, RetryPolicy retryPolicy)
+ throws IOException {
+ RetryPolicy.RetryAction action = null;
+ try {
+ action = retryPolicy.shouldRetry(exception, retryCount, 0, true);
+ } catch (Exception e) {
+ setExceptionAndThrow(new IOException(e));
+ }
+ if (action != null &&
+ action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
+ String msg = "";
+ if (action.reason != null) {
+ msg = "Retry request failed. " + action.reason;
+ }
+ setExceptionAndThrow(new IOException(msg, exception));
+ }
+
+ // Throw the exception if the thread is interrupted
+ if (Thread.currentThread().isInterrupted()) {
+ setExceptionAndThrow(exception);
+ }
+ Preconditions.checkNotNull(action);
+ Preconditions.checkArgument(
+ action.action == RetryPolicy.RetryAction.RetryDecision.RETRY);
+ if (action.delayMillis > 0) {
+ try {
+ Thread.sleep(action.delayMillis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ IOException ioe = (IOException) new InterruptedIOException(
+ "Interrupted: action=" + action + ", retry policy=" + retryPolicy)
+ .initCause(e);
+ setExceptionAndThrow(ioe);
+ }
+ }
+ retryCount++;
+ }
+}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 3df5eb0e12..d5b9dd9d81 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
@@ -44,8 +45,6 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.ratis.client.api.DataStreamOutput;
import org.apache.ratis.io.StandardWriteOption;
import org.apache.ratis.protocol.DataStreamReply;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.protocol.RoutingTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -208,44 +207,13 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
if (isDatastreamPipelineMode) {
return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
.stream(message.getContent().asReadOnlyByteBuffer(),
- getRoutingTable(pipeline));
+ RatisHelper.getRoutingTable(pipeline));
} else {
return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
.stream(message.getContent().asReadOnlyByteBuffer());
}
}
- public RoutingTable getRoutingTable(Pipeline pipeline) {
- RaftPeerId primaryId = null;
- List<RaftPeerId> raftPeers = new ArrayList<>();
-
- for (DatanodeDetails dn : pipeline.getNodes()) {
- final RaftPeerId raftPeerId = RaftPeerId.valueOf(dn.getUuidString());
- try {
- if (dn == pipeline.getFirstNode()) {
- primaryId = raftPeerId;
- }
- } catch (IOException e) {
- LOG.error("Can not get FirstNode from the pipeline: {} with " +
- "exception: {}", pipeline.toString(), e.getLocalizedMessage());
- return null;
- }
- raftPeers.add(raftPeerId);
- }
-
- RoutingTable.Builder builder = RoutingTable.newBuilder();
- RaftPeerId previousId = primaryId;
- for (RaftPeerId peerId : raftPeers) {
- if (peerId.equals(primaryId)) {
- continue;
- }
- builder.addSuccessor(previousId, peerId);
- previousId = peerId;
- }
-
- return builder.build();
- }
-
public BlockID getBlockID() {
return blockID.get();
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index ca3170985e..b381d7a3a1 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -53,6 +53,7 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.protocol.RoutingTable;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
@@ -391,6 +392,37 @@ public final class RatisHelper {
.min(Long::compareTo).orElse(null);
}
+ public static RoutingTable getRoutingTable(Pipeline pipeline) {
+ RaftPeerId primaryId = null;
+ List<RaftPeerId> raftPeers = new ArrayList<>();
+
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ final RaftPeerId raftPeerId = RaftPeerId.valueOf(dn.getUuidString());
+ try {
+ if (dn == pipeline.getFirstNode()) {
+ primaryId = raftPeerId;
+ }
+ } catch (IOException e) {
+ LOG.error("Can not get FirstNode from the pipeline: {} with " +
+ "exception: {}", pipeline.toString(), e.getLocalizedMessage());
+ return null;
+ }
+ raftPeers.add(raftPeerId);
+ }
+
+ RoutingTable.Builder builder = RoutingTable.newBuilder();
+ RaftPeerId previousId = primaryId;
+ for (RaftPeerId peerId : raftPeers) {
+ if (peerId.equals(primaryId)) {
+ continue;
+ }
+ builder.addSuccessor(previousId, peerId);
+ previousId = peerId;
+ }
+
+ return builder.build();
+ }
+
private static <U> Class<? extends U> getClass(String name,
Class<U> xface) {
try {
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index d90ae6173c..d91c2e9116 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -594,6 +594,18 @@ public final class ContainerTestHelper {
return String.format("%1$" + length + "s", string);
}
+ public static byte[] generateData(int length, boolean random) {
+ final byte[] data = new byte[length];
+ if (random) {
+ ThreadLocalRandom.current().nextBytes(data);
+ } else {
+ for (int i = 0; i < length; i++) {
+ data[i] = (byte) i;
+ }
+ }
+ return data;
+ }
+
/**
* Construct fake protobuf messages for various types of requests.
* This is tedious, however necessary to test. Protobuf classes are final
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index e06d9f9fc0..138cddd486 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.utils.BufferUtils;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.util.Time;
@@ -549,19 +550,29 @@ public class ContainerStateMachine extends BaseStateMachine {
@Override
public CompletableFuture<?> link(DataStream stream, LogEntryProto entry) {
- return CompletableFuture.supplyAsync(() -> {
- if (stream == null) {
- return JavaUtils.completeExceptionally(
- new IllegalStateException("DataStream is null"));
- }
- if (stream.getDataChannel().isOpen()) {
- return JavaUtils.completeExceptionally(
- new IllegalStateException(
- "DataStream: " + stream + " is not closed properly"));
- } else {
- return CompletableFuture.completedFuture(null);
+ if (stream == null) {
+ return JavaUtils.completeExceptionally(new IllegalStateException(
+ "DataStream is null"));
+ }
+ final DataChannel dataChannel = stream.getDataChannel();
+ if (dataChannel.isOpen()) {
+ return JavaUtils.completeExceptionally(new IllegalStateException(
+ "DataStream: " + stream + " is not closed properly"));
+ }
+
+ final CompletableFuture<ContainerCommandResponseProto> f;
+ if (dataChannel instanceof KeyValueStreamDataChannel) {
+ f = CompletableFuture.completedFuture(null);
+ } else {
+ return JavaUtils.completeExceptionally(new IllegalStateException(
+ "Unexpected DataChannel " + dataChannel.getClass()));
+ }
+ return f.whenComplete((res, e) -> {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("PutBlock {} Term: {} Index: {}",
+ res.getResult(), entry.getTerm(), entry.getIndex());
}
- }, executor);
+ });
}
private ExecutorService getChunkExecutor(WriteChunkRequestProto req) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index c72d789909..82e52b3d19 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -181,13 +181,21 @@ public class KeyValueHandler extends Handler {
@Override
public StateMachine.DataChannel getStreamDataChannel(
- Container container, ContainerCommandRequestProto msg)
- throws StorageContainerException {
+ Container container, ContainerCommandRequestProto msg)
+ throws StorageContainerException {
KeyValueContainer kvContainer = (KeyValueContainer) container;
checkContainerOpen(kvContainer);
- BlockID blockID = BlockID.getFromProtobuf(msg.getWriteChunk().getBlockID());
- return chunkManager.getStreamDataChannel(kvContainer,
- blockID, metrics);
+
+ if (msg.hasWriteChunk()) {
+ BlockID blockID =
+ BlockID.getFromProtobuf(msg.getWriteChunk().getBlockID());
+
+ return chunkManager.getStreamDataChannel(kvContainer,
+ blockID, metrics);
+ } else {
+ throw new StorageContainerException("Malformed request.",
+ ContainerProtos.Result.IO_EXCEPTION);
+ }
}
@Override
@@ -268,10 +276,14 @@ public class KeyValueHandler extends Handler {
ContainerCommandResponseProto handleStreamInit(
ContainerCommandRequestProto request, KeyValueContainer kvContainer,
DispatcherContext dispatcherContext) {
- if (!request.hasWriteChunk()) {
+ final BlockID blockID;
+ if (request.hasWriteChunk()) {
+ WriteChunkRequestProto writeChunk = request.getWriteChunk();
+ blockID = BlockID.getFromProtobuf(writeChunk.getBlockID());
+ } else {
if (LOG.isDebugEnabled()) {
- LOG.debug("Malformed Write Chunk request. trace ID: {}",
- request.getTraceID());
+ LOG.debug("Malformed {} request. trace ID: {}",
+ request.getCmdType(), request.getTraceID());
}
return malformedRequest(request);
}
@@ -279,13 +291,8 @@ public class KeyValueHandler extends Handler {
String path = null;
try {
checkContainerOpen(kvContainer);
-
- WriteChunkRequestProto writeChunk = request.getWriteChunk();
- BlockID blockID = BlockID.getFromProtobuf(writeChunk.getBlockID());
-
path = chunkManager
.streamInit(kvContainer, blockID);
-
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
index 14ead4ea86..66723031f0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
@@ -28,7 +28,7 @@ import java.io.File;
/**
* This class is used to get the DataChannel for streaming.
*/
-class KeyValueStreamDataChannel extends StreamDataChannelBase {
+public class KeyValueStreamDataChannel extends StreamDataChannelBase {
KeyValueStreamDataChannel(File file, ContainerData containerData,
ContainerMetrics metrics)
throws StorageContainerException {
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index 00cda7844a..e51242cc10 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -18,10 +18,8 @@
*/
package org.apache.hadoop.ozone.client.io;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
@@ -88,30 +86,6 @@ public class BlockDataStreamOutputEntryPool {
this.bufferList = new ArrayList<>();
}
- /**
- * A constructor for testing purpose only.
- *
- * @see KeyDataStreamOutput#KeyDataStreamOutput()
- */
- @VisibleForTesting
- BlockDataStreamOutputEntryPool() {
- streamEntries = new ArrayList<>();
- omClient = null;
- keyArgs = null;
- xceiverClientFactory = null;
- config =
- new OzoneConfiguration().getObject(OzoneClientConfig.class);
- config.setStreamBufferSize(0);
- config.setStreamBufferMaxSize(0);
- config.setStreamBufferFlushSize(0);
- config.setStreamBufferFlushDelay(false);
- requestID = null;
- int chunkSize = 0;
- currentStreamIndex = 0;
- openID = -1;
- excludeList = new ExcludeList();
- }
-
/**
* When a key is opened, it is possible that there are some blocks already
* allocated to it for this open session. In this case, to make use of these
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
index 2540e42e24..dc5c3a016d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
@@ -28,31 +28,22 @@ import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.hdds.scm.storage.AbstractDataStreamOutput;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.ratis.protocol.exceptions.AlreadyClosedException;
-import org.apache.ratis.protocol.exceptions.RaftRetryFailureException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
/**
* Maintaining a list of BlockInputStream. Write based on offset.
@@ -63,7 +54,7 @@ import java.util.stream.Collectors;
*
* TODO : currently not support multi-thread access.
*/
-public class KeyDataStreamOutput implements ByteBufferStreamOutput {
+public class KeyDataStreamOutput extends AbstractDataStreamOutput {
private OzoneClientConfig config;
@@ -79,34 +70,16 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
private boolean closed;
private FileEncryptionInfo feInfo;
- private final Map<Class<? extends Throwable>, RetryPolicy> retryPolicyMap;
- private int retryCount;
+
// how much of data is actually written yet to underlying stream
private long offset;
// how much data has been ingested into the stream
private long writeOffset;
- // whether an exception is encountered while write and whole write could
- // not succeed
- private boolean isException;
+
private final BlockDataStreamOutputEntryPool blockDataStreamOutputEntryPool;
private long clientID;
- /**
- * A constructor for testing purpose only.
- */
- @VisibleForTesting
- public KeyDataStreamOutput() {
- closed = false;
- this.retryPolicyMap = HddsClientUtils.getExceptionList()
- .stream()
- .collect(Collectors.toMap(Function.identity(),
- e -> RetryPolicies.TRY_ONCE_THEN_FAIL));
- retryCount = 0;
- offset = 0;
- blockDataStreamOutputEntryPool = new BlockDataStreamOutputEntryPool();
- }
-
@VisibleForTesting
public List<BlockDataStreamOutputEntry> getStreamEntries() {
return blockDataStreamOutputEntryPool.getStreamEntries();
@@ -122,11 +95,6 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
return blockDataStreamOutputEntryPool.getLocationInfoList();
}
- @VisibleForTesting
- public int getRetryCount() {
- return retryCount;
- }
-
@VisibleForTesting
public long getClientID() {
return clientID;
@@ -142,6 +110,8 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
String uploadID, int partNumber, boolean isMultipart,
boolean unsafeByteBufferConversion
) {
+ super(HddsClientUtils.getRetryPolicyByException(
+ config.getMaxRetryCount(), config.getRetryInterval()));
this.config = config;
OmKeyInfo info = handler.getKeyInfo();
blockDataStreamOutputEntryPool =
@@ -158,10 +128,6 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
// Retrieve the file encryption key info, null if file is not in
// encrypted bucket.
this.feInfo = info.getFileEncryptionInfo();
- this.retryPolicyMap = HddsClientUtils.getRetryPolicyByException(
- config.getMaxRetryCount(), config.getRetryInterval());
- this.retryCount = 0;
- this.isException = false;
this.writeOffset = 0;
this.clientID = handler.getId();
}
@@ -322,9 +288,10 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
if (bufferedDataLen > 0) {
// If the data is still cached in the underlying stream, we need to
// allocate new block and write this data in the datanode.
- handleRetry(exception, bufferedDataLen);
+ handleRetry(exception);
+ handleWrite(null, 0, bufferedDataLen, true);
// reset the retryCount after handling the exception
- retryCount = 0;
+ resetRetryCount();
}
}
@@ -333,74 +300,6 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
closed = true;
}
- private void handleRetry(IOException exception, long len) throws IOException {
- RetryPolicy retryPolicy = retryPolicyMap
- .get(HddsClientUtils.checkForException(exception).getClass());
- if (retryPolicy == null) {
- retryPolicy = retryPolicyMap.get(Exception.class);
- }
- RetryPolicy.RetryAction action = null;
- try {
- action = retryPolicy.shouldRetry(exception, retryCount, 0, true);
- } catch (Exception e) {
- setExceptionAndThrow(new IOException(e));
- }
- if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
- String msg = "";
- if (action.reason != null) {
- msg = "Retry request failed. " + action.reason;
- LOG.error(msg, exception);
- }
- setExceptionAndThrow(new IOException(msg, exception));
- }
-
- // Throw the exception if the thread is interrupted
- if (Thread.currentThread().isInterrupted()) {
- LOG.warn("Interrupted while trying for retry");
- setExceptionAndThrow(exception);
- }
- Preconditions.checkArgument(
- action.action == RetryPolicy.RetryAction.RetryDecision.RETRY);
- if (action.delayMillis > 0) {
- try {
- Thread.sleep(action.delayMillis);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- IOException ioe = (IOException) new InterruptedIOException(
- "Interrupted: action=" + action + ", retry policy=" + retryPolicy)
- .initCause(e);
- setExceptionAndThrow(ioe);
- }
- }
- retryCount++;
- if (LOG.isTraceEnabled()) {
- LOG.trace("Retrying Write request. Already tried {} time(s); " +
- "retry policy is {} ", retryCount, retryPolicy);
- }
- handleWrite(null, 0, len, true);
- }
-
- private void setExceptionAndThrow(IOException ioe) throws IOException {
- isException = true;
- throw ioe;
- }
-
- /**
- * Checks if the provided exception signifies retry failure in ratis client.
- * In case of retry failure, ratis client throws RaftRetryFailureException
- * and all succeeding operations are failed with AlreadyClosedException.
- */
- private boolean checkForRetryFailure(Throwable t) {
- return t instanceof RaftRetryFailureException
- || t instanceof AlreadyClosedException;
- }
-
- // Every container specific exception from datatnode will be seen as
- // StorageContainerException
- private boolean checkIfContainerToExclude(Throwable t) {
- return t instanceof StorageContainerException;
- }
-
@Override
public void flush() throws IOException {
checkNotClosed();
@@ -485,7 +384,7 @@ public class KeyDataStreamOutput implements ByteBufferStreamOutput {
closed = true;
try {
handleFlushOrClose(StreamAction.CLOSE);
- if (!isException) {
+ if (!isException()) {
Preconditions.checkArgument(writeOffset == offset);
}
blockDataStreamOutputEntryPool.commitKey(offset);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 6225e25268..65f7348740 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -161,7 +161,7 @@ public class TestBlockDataStreamOutput {
private void testWrite(int dataLength) throws Exception {
String keyName = getKeyName();
OzoneDataStreamOutput key = createKey(
- keyName, ReplicationType.RATIS, 0);
+ keyName, ReplicationType.RATIS, dataLength);
byte[] data =
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
@@ -174,7 +174,7 @@ public class TestBlockDataStreamOutput {
private void testWriteWithFailure(int dataLength) throws Exception {
String keyName = getKeyName();
OzoneDataStreamOutput key = createKey(
- keyName, ReplicationType.RATIS, 0);
+ keyName, ReplicationType.RATIS, dataLength);
byte[] data =
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
index f4c756bccd..ad9eca6af7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.client.rpc;
-
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -46,10 +46,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.List;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
@@ -74,6 +72,11 @@ public class TestContainerStateMachineStream {
private String volumeName;
private String bucketName;
+ private static final int CHUNK_SIZE = 100;
+ private static final int FLUSH_SIZE = 2 * CHUNK_SIZE;
+ private static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE;
+ private static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE;
+
/**
* Create a MiniDFSCluster for testing.
*
@@ -118,8 +121,15 @@ public class TestContainerStateMachineStream {
conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1);
conf.setQuietMode(false);
cluster =
- MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).setHbInterval(200)
+ MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .setHbInterval(200)
.setDataStreamMinPacketSize(1024)
+ .setBlockSize(BLOCK_SIZE)
+ .setChunkSize(CHUNK_SIZE)
+ .setStreamBufferFlushSize(FLUSH_SIZE)
+ .setStreamBufferMaxSize(MAX_FLUSH_SIZE)
+ .setStreamBufferSizeUnit(StorageUnit.BYTES)
.build();
cluster.waitForClusterToBeReady();
cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000);
@@ -146,20 +156,14 @@ public class TestContainerStateMachineStream {
@Test
public void testContainerStateMachineForStreaming() throws Exception {
- long size = 1024 * 8;
+ long size = CHUNK_SIZE + 1;
OzoneDataStreamOutput key = TestHelper.createStreamKey(
"ozone-stream-test.txt", ReplicationType.RATIS, size, objectStore,
volumeName, bucketName);
- byte[] data =
- ContainerTestHelper
- .getFixedLengthString(UUID.randomUUID().toString(),
- (int) (size / 2))
- .getBytes(UTF_8);
+ byte[] data = ContainerTestHelper.generateData((int) size, true);
key.write(ByteBuffer.wrap(data));
- key.write(ByteBuffer.wrap(data));
-
key.flush();
KeyDataStreamOutput streamOutput =
@@ -181,4 +185,35 @@ public class TestContainerStateMachineStream {
Assert.assertTrue(bytesUsed == size);
}
+
+ @Test
+ public void testContainerStateMachineForStreamingSmallFile()
+ throws Exception {
+ long size = CHUNK_SIZE - 1;
+
+ OzoneDataStreamOutput key = TestHelper.createStreamKey(
+ "ozone-stream-test-small-file.txt", ReplicationType.RATIS, size,
+ objectStore, volumeName, bucketName);
+
+ byte[] data = ContainerTestHelper.generateData((int) size, true);
+ key.write(ByteBuffer.wrap(data));
+ key.flush();
+
+ KeyDataStreamOutput streamOutput =
+ (KeyDataStreamOutput) key.getByteBufStreamOutput();
+ List<OmKeyLocationInfo> locationInfoList =
+ streamOutput.getLocationInfoList();
+ key.close();
+ OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
+ HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
+ cluster);
+
+ long bytesUsed = dn.getDatanodeStateMachine()
+ .getContainer().getContainerSet()
+ .getContainer(omKeyLocationInfo.getContainerID()).
+ getContainerData().getBytesUsed();
+
+ Assert.assertTrue(bytesUsed == size);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 16/35: HDDS-5879. [Ozone-Streaming] OzoneBucket add the createMultipartStreamKey method (#2760)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 22d055346781d0960dcb27bf63f0bf491b9986af
Author: hao guo <gu...@360.cn>
AuthorDate: Fri Nov 19 11:21:55 2021 +0800
HDDS-5879. [Ozone-Streaming] OzoneBucket add the createMultipartStreamKey method (#2760)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 15 +++++
.../ozone/client/protocol/ClientProtocol.java | 18 ++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 64 ++++++++++++++++++++++
.../client/rpc/TestOzoneRpcClientWithRatis.java | 53 ++++++++++++++++++
4 files changed, 150 insertions(+)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 89fe02d678..bf7f618b96 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -807,6 +807,21 @@ public class OzoneBucket extends WithMetadata {
uploadID);
}
+ /**
+ * Create a part key for a multipart upload key.
+ * @param key
+ * @param size
+ * @param partNumber
+ * @param uploadID
+ * @return OzoneDataStreamOutput
+ * @throws IOException
+ */
+ public OzoneDataStreamOutput createMultipartStreamKey(String key,
+ long size, int partNumber, String uploadID) throws IOException {
+ return proxy.createMultipartStreamKey(volumeName, name,
+ key, size, partNumber, uploadID);
+ }
+
/**
* Complete Multipart upload. This will combine all the parts and make the
* key visible in ozone.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index de800b113d..5083f5a78a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -478,6 +478,24 @@ public interface ClientProtocol {
int partNumber, String uploadID)
throws IOException;
+ /**
+ * Create a part key for a multipart upload key.
+ * @param volumeName
+ * @param bucketName
+ * @param keyName
+ * @param size
+ * @param partNumber
+ * @param uploadID
+ * @return OzoneDataStreamOutput
+ * @throws IOException
+ */
+ OzoneDataStreamOutput createMultipartStreamKey(String volumeName,
+ String bucketName,
+ String keyName, long size,
+ int partNumber,
+ String uploadID)
+ throws IOException;
+
/**
* Complete Multipart upload. This will combine all the parts and make the
* key visible in ozone.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 4d8ed2dd91..c3002a5abc 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1368,6 +1368,70 @@ public class RpcClient implements ClientProtocol {
}
}
+ @Override
+ public OzoneDataStreamOutput createMultipartStreamKey(
+ String volumeName,
+ String bucketName,
+ String keyName,
+ long size,
+ int partNumber,
+ String uploadID)
+ throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ if (checkKeyNameEnabled) {
+ HddsClientUtils.verifyKeyName(keyName);
+ }
+ HddsClientUtils.checkNotNull(keyName, uploadID);
+ Preconditions.checkArgument(partNumber > 0 && partNumber <= 10000, "Part " +
+ "number should be greater than zero and less than or equal to 10000");
+ Preconditions.checkArgument(size >= 0, "size should be greater than or " +
+ "equal to zero");
+ String requestId = UUID.randomUUID().toString();
+
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(size)
+ .setIsMultipartKey(true)
+ .setMultipartUploadID(uploadID)
+ .setMultipartUploadPartNumber(partNumber)
+ .setAcls(getAclList())
+ .build();
+
+ OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
+
+ KeyDataStreamOutput keyOutputStream =
+ new KeyDataStreamOutput.Builder()
+ .setHandler(openKey)
+ .setXceiverClientManager(xceiverClientManager)
+ .setOmClient(ozoneManagerClient)
+ .setRequestID(requestId)
+ .setReplicationConfig(openKey.getKeyInfo().getReplicationConfig())
+ .setMultipartNumber(partNumber)
+ .setMultipartUploadID(uploadID)
+ .setIsMultipartKey(true)
+ .enableUnsafeByteBufferConversion(unsafeByteBufferConversion)
+ .setConfig(clientConfig)
+ .build();
+ keyOutputStream
+ .addPreallocateBlocks(
+ openKey.getKeyInfo().getLatestVersionLocations(),
+ openKey.getOpenVersion());
+
+ FileEncryptionInfo feInfo = openKey.getKeyInfo().getFileEncryptionInfo();
+ if (feInfo != null) {
+ // todo: need to support file encrypt,
+ // https://issues.apache.org/jira/browse/HDDS-5892
+ throw new UnsupportedOperationException(
+ "FileEncryptionInfo is not yet supported in " +
+ "createMultipartStreamKey");
+ } else {
+ return new OzoneDataStreamOutput(keyOutputStream);
+ }
+ }
+
@Override
public OmMultipartUploadCompleteInfo completeMultipartUpload(
String volumeName, String bucketName, String keyName, String uploadID,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithRatis.java
index 362a218af2..d9c67609e6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithRatis.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithRatis.java
@@ -19,10 +19,12 @@
package org.apache.hadoop.ozone.client.rpc;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.UUID;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -31,12 +33,15 @@ import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.junit.jupiter.api.AfterAll;
import org.junit.Assert;
import org.junit.jupiter.api.BeforeAll;
@@ -44,6 +49,7 @@ import org.junit.jupiter.api.Test;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
/**
@@ -155,4 +161,51 @@ public class TestOzoneRpcClientWithRatis extends TestOzoneRpcClientAbstract {
}
}
}
+
+ @Test
+ public void testMultiPartUploadWithStream() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+
+ byte[] sampleData = new byte[1024 * 8];
+
+ int valueLength = sampleData.length;
+
+ getStore().createVolume(volumeName);
+ OzoneVolume volume = getStore().getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ ReplicationConfig replicationConfig =
+ ReplicationConfig.fromTypeAndFactor(
+ ReplicationType.RATIS,
+ THREE);
+
+ OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName,
+ replicationConfig);
+
+ assertNotNull(multipartInfo);
+ String uploadID = multipartInfo.getUploadID();
+ Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
+ Assert.assertEquals(bucketName, multipartInfo.getBucketName());
+ Assert.assertEquals(keyName, multipartInfo.getKeyName());
+ assertNotNull(multipartInfo.getUploadID());
+
+ OzoneDataStreamOutput ozoneStreamOutput = bucket.createMultipartStreamKey(
+ keyName, valueLength, 1, uploadID);
+ ozoneStreamOutput.write(ByteBuffer.wrap(sampleData), 0,
+ valueLength);
+ ozoneStreamOutput.close();
+
+ OzoneMultipartUploadPartListParts parts =
+ bucket.listParts(keyName, uploadID, 0, 1);
+
+ Assert.assertEquals(parts.getPartInfoList().size(), 1);
+
+ OzoneMultipartUploadPartListParts.PartInfo partInfo =
+ parts.getPartInfoList().get(0);
+ Assert.assertEquals(valueLength, partInfo.getSize());
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 09/35: HDDS-5486. [Ozone-Streaming] Streaming supports writing in Pipline mode (#2682)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7663ee5766a25c16efbc42583694987188fd5986
Author: micah zhao <mi...@tencent.com>
AuthorDate: Thu Sep 30 11:21:47 2021 +0800
HDDS-5486. [Ozone-Streaming] Streaming supports writing in Pipline mode (#2682)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 40 ++++++++++++++++++++--
1 file changed, 37 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index c69af90a91..41e2c48bbb 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.ratis.client.api.DataStreamOutput;
import org.apache.ratis.io.StandardWriteOption;
import org.apache.ratis.protocol.DataStreamReply;
+import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.protocol.RoutingTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,7 +146,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
this.xceiverClient =
(XceiverClientRatis)xceiverClientManager.acquireClient(pipeline);
// Alternatively, stream setup can be delayed till the first chunk write.
- this.out = setupStream();
+ this.out = setupStream(pipeline);
this.token = token;
flushPeriod = (int) (config.getStreamBufferFlushSize() / config
@@ -166,7 +168,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
config.getBytesPerChecksum());
}
- private DataStreamOutput setupStream() throws IOException {
+ private DataStreamOutput setupStream(Pipeline pipeline) throws IOException {
// Execute a dummy WriteChunk request to get the path of the target file,
// but does NOT write any data to it.
ContainerProtos.WriteChunkRequestProto.Builder writeChunkRequest =
@@ -184,7 +186,39 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
ContainerCommandRequestMessage.toMessage(builder.build(), null);
return Preconditions.checkNotNull(xceiverClient.getDataStreamApi())
- .stream(message.getContent().asReadOnlyByteBuffer());
+ .stream(message.getContent().asReadOnlyByteBuffer(),
+ getRoutingTable(pipeline));
+ }
+
+ public RoutingTable getRoutingTable(Pipeline pipeline) {
+ RaftPeerId primaryId = null;
+ List<RaftPeerId> raftPeers = new ArrayList<>();
+
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ final RaftPeerId raftPeerId = RaftPeerId.valueOf(dn.getUuidString());
+ try {
+ if (dn == pipeline.getFirstNode()) {
+ primaryId = raftPeerId;
+ }
+ } catch (IOException e) {
+ LOG.error("Can not get FirstNode from the pipeline: {} with " +
+ "exception: {}", pipeline.toString(), e.getLocalizedMessage());
+ return null;
+ }
+ raftPeers.add(raftPeerId);
+ }
+
+ RoutingTable.Builder builder = RoutingTable.newBuilder();
+ RaftPeerId previousId = primaryId;
+ for (RaftPeerId peerId : raftPeers) {
+ if (peerId.equals(primaryId)) {
+ continue;
+ }
+ builder.addSuccessor(previousId, peerId);
+ previousId = peerId;
+ }
+
+ return builder.build();
}
public BlockID getBlockID() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 15/35: HDDS-5961. [Ozone-Streaming] update the usage space of Containers in the stream write (#2833)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit bb41750fdd4945699a6dfe7390f3457bfe28fd06
Author: hao guo <gu...@360.cn>
AuthorDate: Wed Nov 17 15:19:47 2021 +0800
HDDS-5961. [Ozone-Streaming] update the usage space of Containers in the stream write (#2833)
---
.../container/common/impl/HddsDispatcher.java | 18 ++
.../common/interfaces/ContainerDispatcher.java | 10 ++
.../ozone/container/common/interfaces/Handler.java | 5 +
.../server/ratis/ContainerStateMachine.java | 21 ++-
.../transport/server/ratis/StreamDataChannel.java | 57 -------
.../ozone/container/keyvalue/KeyValueHandler.java | 12 ++
.../keyvalue/impl/ChunkManagerDispatcher.java | 10 ++
.../keyvalue/impl/FilePerBlockStrategy.java | 12 ++
.../keyvalue/impl/KeyValueStreamDataChannel.java | 90 ++++++++++
.../keyvalue/interfaces/ChunkManager.java | 8 +
.../src/main/proto/DatanodeClientProtocol.proto | 1 +
.../rpc/TestContainerStateMachineStream.java | 183 +++++++++++++++++++++
12 files changed, 364 insertions(+), 63 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 2e0f1de496..e81308a4f3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -66,6 +66,7 @@ import com.google.protobuf.ServiceException;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
+import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.thirdparty.com.google.protobuf.ProtocolMessageEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -675,4 +676,21 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
default: return false;
}
}
+
+ @Override
+ public StateMachine.DataChannel getStreamDataChannel(
+ ContainerCommandRequestProto msg)
+ throws StorageContainerException {
+ long containerID = msg.getContainerID();
+ Container container = getContainer(containerID);
+ if (container != null) {
+ Handler handler = getHandler(getContainerType(container));
+ return handler.getStreamDataChannel(container, msg);
+ } else {
+ throw new StorageContainerException(
+ "ContainerID " + containerID + " does not exist",
+ ContainerProtos.Result.CONTAINER_NOT_FOUND);
+ }
+ }
+
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
index a2e397d546..d02bae0a35 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
+import org.apache.ratis.statemachine.StateMachine;
import java.util.Map;
@@ -84,4 +85,13 @@ public interface ContainerDispatcher {
* @param clusterId
*/
void setClusterId(String clusterId);
+
+ /**
+ * When uploading using stream, get StreamDataChannel.
+ */
+ default StateMachine.DataChannel getStreamDataChannel(
+ ContainerCommandRequestProto msg) throws StorageContainerException {
+ throw new UnsupportedOperationException(
+ "getStreamDataChannel not supported.");
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 1dbd588d33..7401aa545d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.Dispatche
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import org.apache.ratis.statemachine.StateMachine;
/**
* Dispatcher sends ContainerCommandRequests to Handler. Each Container Type
@@ -80,6 +81,10 @@ public abstract class Handler {
}
}
+ public abstract StateMachine.DataChannel getStreamDataChannel(
+ Container container, ContainerCommandRequestProto msg)
+ throws StorageContainerException;
+
/**
* Returns the Id of this datanode.
*
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index ed5659d1d7..e06d9f9fc0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collection;
-import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@@ -82,6 +81,7 @@ import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.storage.RaftStorage;
+import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.statemachine.StateMachineStorage;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
@@ -513,6 +513,19 @@ public class ContainerStateMachine extends BaseStateMachine {
return raftFuture;
}
+ private StateMachine.DataChannel getStreamDataChannel(
+ ContainerCommandRequestProto requestProto,
+ DispatcherContext context) throws StorageContainerException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: getStreamDataChannel {} containerID={} pipelineID={} " +
+ "traceID={}", gid, requestProto.getCmdType(),
+ requestProto.getContainerID(), requestProto.getPipelineID(),
+ requestProto.getTraceID());
+ }
+ runCommand(requestProto, context); // stream init
+ return dispatcher.getStreamDataChannel(requestProto);
+ }
+
@Override
public CompletableFuture<DataStream> stream(RaftClientRequest request) {
return CompletableFuture.supplyAsync(() -> {
@@ -524,11 +537,7 @@ public class ContainerStateMachine extends BaseStateMachine {
.setStage(DispatcherContext.WriteChunkStage.WRITE_DATA)
.setContainer2BCSIDMap(container2BCSIDMap)
.build();
-
- ContainerCommandResponseProto response = runCommand(
- requestProto, context);
- final StreamDataChannel channel = new StreamDataChannel(
- Paths.get(response.getMessage()));
+ DataChannel channel = getStreamDataChannel(requestProto, context);
final ExecutorService chunkExecutor = requestProto.hasWriteChunk() ?
getChunkExecutor(requestProto.getWriteChunk()) : null;
return new LocalStream(channel, chunkExecutor);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java
deleted file mode 100644
index 3df66e26dc..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/StreamDataChannel.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-import org.apache.ratis.statemachine.StateMachine;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-
-class StreamDataChannel implements StateMachine.DataChannel {
- private final Path path;
- private final RandomAccessFile randomAccessFile;
-
- StreamDataChannel(Path path) throws FileNotFoundException {
- this.path = path;
- this.randomAccessFile = new RandomAccessFile(path.toFile(), "rw");
- }
-
- @Override
- public void force(boolean metadata) throws IOException {
- randomAccessFile.getChannel().force(metadata);
- }
-
- @Override
- public int write(ByteBuffer src) throws IOException {
- return randomAccessFile.getChannel().write(src);
- }
-
- @Override
- public boolean isOpen() {
- return randomAccessFile.getChannel().isOpen();
- }
-
- @Override
- public void close() throws IOException {
- randomAccessFile.close();
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 44e5f38a90..c72d789909 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -105,6 +105,7 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuil
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion;
+import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,6 +179,17 @@ public class KeyValueHandler extends Handler {
return volumeChoosingPolicy;
}
+ @Override
+ public StateMachine.DataChannel getStreamDataChannel(
+ Container container, ContainerCommandRequestProto msg)
+ throws StorageContainerException {
+ KeyValueContainer kvContainer = (KeyValueContainer) container;
+ checkContainerOpen(kvContainer);
+ BlockID blockID = BlockID.getFromProtobuf(msg.getWriteChunk().getBlockID());
+ return chunkManager.getStreamDataChannel(kvContainer,
+ blockID, metrics);
+ }
+
@Override
public void stop() {
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
index 3e2ab46470..92f6327447 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
@@ -33,6 +34,7 @@ import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.ratis.statemachine.StateMachine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,6 +81,14 @@ public class ChunkManagerDispatcher implements ChunkManager {
.streamInit(container, blockID);
}
+ @Override
+ public StateMachine.DataChannel getStreamDataChannel(
+ Container container, BlockID blockID, ContainerMetrics metrics)
+ throws StorageContainerException {
+ return selectHandler(container)
+ .getStreamDataChannel(container, blockID, metrics);
+ }
+
@Override
public void finishWriteChunks(KeyValueContainer kvContainer,
BlockData blockData) throws IOException {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
index 9efc6bc351..23db342da0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.common.utils.BufferUtils;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -42,6 +43,7 @@ import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.ratis.statemachine.StateMachine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,6 +99,16 @@ public class FilePerBlockStrategy implements ChunkManager {
return chunkFile.getAbsolutePath();
}
+ @Override
+ public StateMachine.DataChannel getStreamDataChannel(
+ Container container, BlockID blockID, ContainerMetrics metrics)
+ throws StorageContainerException {
+ checkLayoutVersion(container);
+ File chunkFile = getChunkFile(container, blockID, null);
+ return new KeyValueStreamDataChannel(chunkFile,
+ container.getContainerData(), metrics);
+ }
+
@Override
public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
ChunkBuffer data, DispatcherContext dispatcherContext)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
new file mode 100644
index 0000000000..c0570f5d4d
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.ratis.statemachine.StateMachine;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+
+/**
+ * This class is used to get the DataChannel for streaming.
+ */
+class KeyValueStreamDataChannel implements StateMachine.DataChannel {
+ private final RandomAccessFile randomAccessFile;
+ private final File file;
+
+ private final ContainerData containerData;
+ private final ContainerMetrics metrics;
+
+ KeyValueStreamDataChannel(File file, ContainerData containerData,
+ ContainerMetrics metrics)
+ throws StorageContainerException {
+ try {
+ this.file = file;
+ this.randomAccessFile = new RandomAccessFile(file, "rw");
+ } catch (FileNotFoundException e) {
+ throw new StorageContainerException("BlockFile not exists with " +
+ "container Id " + containerData.getContainerID() +
+ " file " + file.getAbsolutePath(),
+ ContainerProtos.Result.IO_EXCEPTION);
+ }
+ this.containerData = containerData;
+ this.metrics = metrics;
+ }
+
+ @Override
+ public void force(boolean metadata) throws IOException {
+ randomAccessFile.getChannel().force(metadata);
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ int writeBytes = randomAccessFile.getChannel().write(src);
+ metrics
+ .incContainerBytesStats(ContainerProtos.Type.StreamWrite, writeBytes);
+ containerData.updateWriteStats(writeBytes, false);
+ return writeBytes;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return randomAccessFile.getChannel().isOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ randomAccessFile.close();
+ }
+
+ @Override
+ public String toString() {
+ return "KeyValueStreamDataChannel{" +
+ "File=" + file.getAbsolutePath() +
+ ", containerID=" + containerData.getContainerID() +
+ '}';
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index ba06eebd69..7a64f07628 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -25,9 +25,11 @@ import org.apache.hadoop.ozone.common.ChecksumData;
import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.ratis.statemachine.StateMachine;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -109,6 +111,12 @@ public interface ChunkManager {
return null;
}
+ default StateMachine.DataChannel getStreamDataChannel(
+ Container container, BlockID blockID, ContainerMetrics metrics)
+ throws StorageContainerException {
+ return null;
+ }
+
static long getBufferCapacityForChunkRead(ChunkInfo chunkInfo,
long defaultReadBufferCapacity) {
long bufferCapacity = 0;
diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
index 62e8b9a55e..b87a0e8df5 100644
--- a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto
@@ -102,6 +102,7 @@ enum Type {
GetCommittedBlockLength = 18;
StreamInit = 19;
+ StreamWrite = 20;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
new file mode 100644
index 0000000000..3b17450376
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.client.rpc;
+
+
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+
+/**
+ * Tests the containerStateMachine stream handling.
+ */
+public class TestContainerStateMachineStream {
+
+ /**
+ * Set a timeout for each test.
+ */
+ @Rule
+ public Timeout timeout = Timeout.seconds(300);
+
+ private MiniOzoneCluster cluster;
+ private OzoneConfiguration conf = new OzoneConfiguration();
+ private OzoneClient client;
+ private ObjectStore objectStore;
+ private String volumeName;
+ private String bucketName;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ *
+ * @throws IOException
+ */
+ @Before
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setStreamBufferFlushDelay(false);
+ conf.setFromObject(clientConfig);
+
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 200,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS);
+ conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1,
+ TimeUnit.SECONDS);
+
+ RatisClientConfig ratisClientConfig =
+ conf.getObject(RatisClientConfig.class);
+ ratisClientConfig.setWriteRequestTimeout(Duration.ofSeconds(10));
+ ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(10));
+ conf.setFromObject(ratisClientConfig);
+
+ DatanodeRatisServerConfig ratisServerConfig =
+ conf.getObject(DatanodeRatisServerConfig.class);
+ ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
+ ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10));
+ conf.setFromObject(ratisServerConfig);
+
+ RatisClientConfig.RaftConfig raftClientConfig =
+ conf.getObject(RatisClientConfig.RaftConfig.class);
+ raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
+ raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10));
+ conf.setFromObject(raftClientConfig);
+
+ conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1);
+ conf.setQuietMode(false);
+ cluster =
+ MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).setHbInterval(200)
+ .build();
+ cluster.waitForClusterToBeReady();
+ cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000);
+ //the easiest way to create an open container is creating a key
+ client = OzoneClientFactory.getRpcClient(conf);
+ objectStore = client.getObjectStore();
+
+ volumeName = "testcontainerstatemachinestream";
+ bucketName = "teststreambucket";
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testContainerStateMachineForStreaming() throws Exception {
+ long size = 1024 * 8;
+
+ OzoneDataStreamOutput key = TestHelper.createStreamKey(
+ "ozone-stream-test.txt", ReplicationType.RATIS, size, objectStore,
+ volumeName, bucketName);
+
+ byte[] data =
+ ContainerTestHelper
+ .getFixedLengthString(UUID.randomUUID().toString(),
+ (int) (size / 2))
+ .getBytes(UTF_8);
+ key.write(ByteBuffer.wrap(data));
+ key.write(ByteBuffer.wrap(data));
+
+ key.flush();
+
+ KeyDataStreamOutput streamOutput =
+ (KeyDataStreamOutput) key.getByteBufStreamOutput();
+ List<OmKeyLocationInfo> locationInfoList =
+ streamOutput.getLocationInfoList();
+
+ key.close();
+
+ OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
+ HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
+ cluster);
+
+ long bytesUsed = dn.getDatanodeStateMachine()
+ .getContainer().getContainerSet()
+ .getContainer(omKeyLocationInfo.getContainerID()).
+ getContainerData().getBytesUsed();
+
+ Assert.assertTrue(bytesUsed == size);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 29/35: HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c5a9686ee64e195d6c80a493bc340f50052a8c7d
Author: hao guo <gu...@360.cn>
AuthorDate: Mon Feb 21 21:02:50 2022 +0800
HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
---
.../apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 10 +++++-----
.../java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java | 4 ++--
.../hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java | 4 ++--
.../java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java | 4 ++--
.../java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java | 4 ++--
.../java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java | 2 +-
.../org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 2 +-
9 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 8dd9e6b50e..8b3e32cf41 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -283,7 +283,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
private void writeChunkIfNeeded() throws IOException {
- if (currentBuffer.length()==0) {
+ if (currentBuffer.length() == 0) {
writeChunk(currentBuffer);
currentBuffer = null;
}
@@ -302,7 +302,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
}
private void allocateNewBufferIfNeeded() {
- if (currentBuffer==null) {
+ if (currentBuffer == null) {
currentBuffer =
StreamBuffer.allocate(config.getDataStreamMinPacketSize());
}
@@ -323,7 +323,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
updateFlushLength();
executePutBlock(false, false);
}
- if (bufferList.size()==streamWindow){
+ if (bufferList.size() == streamWindow) {
try {
checkOpen();
if (!putBlockFutures.isEmpty()) {
@@ -514,7 +514,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
// here, we just limit this buffer to the current position. So that next
// write will happen in new buffer
- if (currentBuffer!=null) {
+ if (currentBuffer != null) {
writeChunk(currentBuffer);
currentBuffer = null;
}
@@ -693,7 +693,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
boolean processExecutionException)
throws IOException {
LOG.error("Command execution was interrupted.");
- if(processExecutionException) {
+ if (processExecutionException) {
handleExecutionException(ex);
} else {
throw new IOException(EXCEPTION_MSG + ex.toString(), ex);
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
index 5118ea5ead..d34e4dca94 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
@@ -48,11 +48,11 @@ public class StreamBuffer {
}
- public void put(StreamBuffer sb){
+ public void put(StreamBuffer sb) {
buffer.put(sb.buffer);
}
- public static StreamBuffer allocate(int size){
+ public static StreamBuffer allocate(int size) {
return new StreamBuffer(ByteBuffer.allocate(size));
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index 24a046f623..00cda7844a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -307,8 +307,8 @@ public class BlockDataStreamOutputEntryPool {
}
long computeBufferData() {
- long totalDataLen =0;
- for (StreamBuffer b : bufferList){
+ long totalDataLen = 0;
+ for (StreamBuffer b : bufferList) {
totalDataLen += b.position();
}
return totalDataLen;
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 42891c492b..19dfa3b6a5 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1011,7 +1011,7 @@ public class RpcClient implements ClientProtocol {
.setAcls(getAclList());
if (Boolean.parseBoolean(metadata.get(OzoneConsts.GDPR_FLAG))) {
- try{
+ try {
GDPRSymmetricKey gKey = new GDPRSymmetricKey(new SecureRandom());
builder.addAllMetadata(gKey.getKeyDetails());
} catch (Exception e) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index dc7a59b30a..11b7e7ede0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -323,8 +323,8 @@ public interface MiniOzoneCluster {
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
protected Optional<Long> streamBufferFlushSize = Optional.empty();
- protected Optional<Long> dataStreamBufferFlushSize= Optional.empty();
- protected Optional<Long> datastreamWindowSize= Optional.empty();
+ protected Optional<Long> dataStreamBufferFlushSize = Optional.empty();
+ protected Optional<Long> datastreamWindowSize = Optional.empty();
protected Optional<Long> streamBufferMaxSize = Optional.empty();
protected OptionalInt dataStreamMinPacketSize = OptionalInt.empty();
protected Optional<Long> blockSize = Optional.empty();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 66a8eef564..4b645f1183 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -665,7 +665,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get());
}
if (!dataStreamMinPacketSize.isPresent()) {
- dataStreamMinPacketSize = OptionalInt.of(chunkSize.get()/4);
+ dataStreamMinPacketSize = OptionalInt.of(chunkSize.get() / 4);
}
if (!datastreamWindowSize.isPresent()) {
datastreamWindowSize = Optional.of((long) 8 * chunkSize.get());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index 21003374d7..6225e25268 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -107,7 +107,7 @@ public class TestBlockDataStreamOutput {
.setDataStreamBufferFlushize(maxFlushSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
.setDataStreamMinPacketSize(chunkSize)
- .setDataStreamStreamWindowSize(5*chunkSize)
+ .setDataStreamStreamWindowSize(5 * chunkSize)
.build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
@@ -137,7 +137,7 @@ public class TestBlockDataStreamOutput {
@Test
public void testHalfChunkWrite() throws Exception {
testWrite(chunkSize / 2);
- testWriteWithFailure(chunkSize/2);
+ testWriteWithFailure(chunkSize / 2);
}
@Test
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index e669f1c5cc..3ac013a837 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -263,7 +263,7 @@ public class BasicOzoneFileSystem extends FileSystem {
boolean isRatisStreamingEnabled = getConf().getBoolean(
OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
- if (isRatisStreamingEnabled){
+ if (isRatisStreamingEnabled) {
return new FSDataOutputStream(adapter.createStreamFile(key,
replication, overwrite, recursive), statistics);
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 0bed09ba8a..600abdf4f1 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -239,7 +239,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
boolean isRatisStreamingEnabled = getConf().getBoolean(
OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
- if (isRatisStreamingEnabled){
+ if (isRatisStreamingEnabled) {
return new FSDataOutputStream(adapter.createStreamFile(key,
replication, overwrite, recursive), statistics);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 21/35: HDDS-6139. [Ozone-Streaming] Fix incorrect computation of totalAckDataLength. (#2978)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 6925126ddc8cef4b64b7180c9028311b4aa5a6fc
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Thu Jan 13 11:24:12 2022 +0530
HDDS-6139. [Ozone-Streaming] Fix incorrect computation of totalAckDataLength. (#2978)
---
.../hdds/scm/storage/BlockDataStreamOutput.java | 9 ++++++++-
.../ozone/client/io/BlockDataStreamOutputEntry.java | 2 +-
.../ozone/client/rpc/TestBlockDataStreamOutput.java | 19 +++++++++++++++++++
3 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index 84968b68d5..6ef59dd6d8 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -133,6 +133,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
private long syncPosition = 0;
private StreamBuffer currentBuffer;
private XceiverClientMetrics metrics;
+ // buffers for which putBlock is yet to be executed
+ private List<StreamBuffer> buffersForPutBlock;
/**
* Creates a new BlockDataStreamOutput.
*
@@ -287,6 +289,10 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
private void writeChunk(StreamBuffer sb) throws IOException {
bufferList.add(sb);
+ if (buffersForPutBlock == null) {
+ buffersForPutBlock = new ArrayList<>();
+ }
+ buffersForPutBlock.add(sb);
ByteBuffer dup = sb.duplicate();
dup.position(0);
dup.limit(sb.position());
@@ -392,7 +398,8 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
final List<StreamBuffer> byteBufferList;
if (!force) {
Preconditions.checkNotNull(bufferList);
- byteBufferList = bufferList;
+ byteBufferList = buffersForPutBlock;
+ buffersForPutBlock = null;
Preconditions.checkNotNull(byteBufferList);
} else {
byteBufferList = null;
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
index 2cd5630549..4e5a35a539 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
@@ -156,7 +156,7 @@ public final class BlockDataStreamOutputEntry
}
}
- long getTotalAckDataLength() {
+ public long getTotalAckDataLength() {
if (byteBufferStreamOutput != null) {
BlockDataStreamOutput out =
(BlockDataStreamOutput) this.byteBufferStreamOutput;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index c9242df8b1..c6a3c32d2b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.BlockDataStreamOutputEntry;
import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -256,4 +257,22 @@ public class TestBlockDataStreamOutput {
validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}
+ @Test
+ public void testTotalAckDataLength() throws Exception {
+ int dataLength = 400;
+ String keyName = getKeyName();
+ OzoneDataStreamOutput key = createKey(
+ keyName, ReplicationType.RATIS, 0);
+ byte[] data =
+ ContainerTestHelper.getFixedLengthString(keyString, dataLength)
+ .getBytes(UTF_8);
+ KeyDataStreamOutput keyDataStreamOutput =
+ (KeyDataStreamOutput) key.getByteBufStreamOutput();
+ BlockDataStreamOutputEntry stream =
+ keyDataStreamOutput.getStreamEntries().get(0);
+ key.write(ByteBuffer.wrap(data));
+ key.close();
+ Assert.assertEquals(dataLength, stream.getTotalAckDataLength());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 08/35: HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7b713b2b5fae3d94d25c0ab00a84278681237ff4
Author: micah zhao <mi...@tencent.com>
AuthorDate: Thu Sep 23 11:26:43 2021 +0800
HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
---
.../org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java | 3 +--
.../src/main/java/org/apache/hadoop/ozone/common/Checksum.java | 5 +++++
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
index d0419fa0c3..c69af90a91 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -485,8 +485,7 @@ public class BlockDataStreamOutput implements ByteBufferStreamOutput {
throws IOException {
final int effectiveChunkSize = buf.remaining();
final long offset = chunkOffset.getAndAdd(effectiveChunkSize);
- ChecksumData checksumData =
- checksum.computeChecksum(buf.asReadOnlyBuffer());
+ ChecksumData checksumData = checksum.computeChecksum(buf);
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(blockID.get().getLocalID() + "_chunk_" + ++chunkIndex)
.setOffset(offset)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
index 76f84c46ab..d300b9ef0e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
@@ -139,6 +139,11 @@ public class Checksum {
*/
public ChecksumData computeChecksum(ByteBuffer data)
throws OzoneChecksumException {
+ // If type is set to NONE, we do not need to compute the checksums. We also
+ // need to avoid unnecessary conversions.
+ if (checksumType == ChecksumType.NONE) {
+ return new ChecksumData(checksumType, bytesPerChecksum);
+ }
if (!data.isReadOnly()) {
data = data.asReadOnlyBuffer();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org